2024-11-27 12:23:55,706 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-27 12:23:55,717 main DEBUG Took 0.009392 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-27 12:23:55,717 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-27 12:23:55,718 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-27 12:23:55,718 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-27 12:23:55,719 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-27 12:23:55,726 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-27 12:23:55,737 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-27 12:23:55,738 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-27 12:23:55,739 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-27 12:23:55,740 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-27 12:23:55,740 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-27 12:23:55,741 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-27 12:23:55,741 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-27 12:23:55,742 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-27 12:23:55,742 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-27 12:23:55,743 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-27 12:23:55,743 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-27 12:23:55,744 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-27 12:23:55,744 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-27 12:23:55,744 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-27 12:23:55,745 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-27 12:23:55,745 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-27 12:23:55,746 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-27 12:23:55,746 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-27 12:23:55,746 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-27 12:23:55,746 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-27 12:23:55,747 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-27 12:23:55,747 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-27 12:23:55,748 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-27 12:23:55,748 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-27 12:23:55,748 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-27 12:23:55,748 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-27 12:23:55,750 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-27 12:23:55,751 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-27 12:23:55,753 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-27 12:23:55,753 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-27 12:23:55,755 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-27 12:23:55,755 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-27 12:23:55,763 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-27 12:23:55,765 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-27 12:23:55,767 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-27 12:23:55,767 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-27 12:23:55,768 main DEBUG createAppenders(={Console}) 2024-11-27 12:23:55,768 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad initialized 2024-11-27 12:23:55,769 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-27 12:23:55,769 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad OK. 2024-11-27 12:23:55,769 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-27 12:23:55,770 main DEBUG OutputStream closed 2024-11-27 12:23:55,770 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-27 12:23:55,770 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-27 12:23:55,770 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@55634720 OK 2024-11-27 12:23:55,837 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-27 12:23:55,839 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-27 12:23:55,840 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-27 12:23:55,841 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-27 12:23:55,841 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-27 12:23:55,841 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-27 12:23:55,842 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-27 12:23:55,842 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-27 12:23:55,842 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-27 12:23:55,843 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-27 12:23:55,843 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-27 12:23:55,843 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-27 12:23:55,843 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-27 12:23:55,844 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-27 12:23:55,844 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-27 12:23:55,844 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-27 12:23:55,845 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-27 12:23:55,845 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-27 12:23:55,847 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-27 12:23:55,848 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@19835e64) with optional ClassLoader: null 2024-11-27 12:23:55,848 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-27 12:23:55,849 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@19835e64] started OK. 2024-11-27T12:23:56,053 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/16bc5e44-ed69-d7f1-cacd-0b5be2f2db9b 2024-11-27 12:23:56,056 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-27 12:23:56,056 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-27T12:23:56,064 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.TestAcidGuaranteesWithNoInMemCompaction timeout: 13 mins 2024-11-27T12:23:56,084 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-27T12:23:56,100 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/16bc5e44-ed69-d7f1-cacd-0b5be2f2db9b/cluster_2674ffaa-0f8a-9abb-ffed-bcc8bba3b369, deleteOnExit=true 2024-11-27T12:23:56,100 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-27T12:23:56,101 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/16bc5e44-ed69-d7f1-cacd-0b5be2f2db9b/test.cache.data in system properties and HBase conf 2024-11-27T12:23:56,102 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/16bc5e44-ed69-d7f1-cacd-0b5be2f2db9b/hadoop.tmp.dir in system properties and HBase conf 2024-11-27T12:23:56,103 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/16bc5e44-ed69-d7f1-cacd-0b5be2f2db9b/hadoop.log.dir in system properties and HBase conf 2024-11-27T12:23:56,103 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/16bc5e44-ed69-d7f1-cacd-0b5be2f2db9b/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-27T12:23:56,104 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/16bc5e44-ed69-d7f1-cacd-0b5be2f2db9b/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-27T12:23:56,104 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-27T12:23:56,187 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-27T12:23:56,293 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-27T12:23:56,297 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/16bc5e44-ed69-d7f1-cacd-0b5be2f2db9b/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-27T12:23:56,298 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/16bc5e44-ed69-d7f1-cacd-0b5be2f2db9b/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-27T12:23:56,298 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/16bc5e44-ed69-d7f1-cacd-0b5be2f2db9b/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-27T12:23:56,299 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/16bc5e44-ed69-d7f1-cacd-0b5be2f2db9b/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-27T12:23:56,299 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/16bc5e44-ed69-d7f1-cacd-0b5be2f2db9b/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-27T12:23:56,300 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/16bc5e44-ed69-d7f1-cacd-0b5be2f2db9b/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-27T12:23:56,300 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/16bc5e44-ed69-d7f1-cacd-0b5be2f2db9b/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-27T12:23:56,301 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/16bc5e44-ed69-d7f1-cacd-0b5be2f2db9b/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-27T12:23:56,301 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/16bc5e44-ed69-d7f1-cacd-0b5be2f2db9b/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-27T12:23:56,302 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/16bc5e44-ed69-d7f1-cacd-0b5be2f2db9b/nfs.dump.dir in system properties and HBase conf 2024-11-27T12:23:56,302 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/16bc5e44-ed69-d7f1-cacd-0b5be2f2db9b/java.io.tmpdir in system properties and HBase conf 2024-11-27T12:23:56,302 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/16bc5e44-ed69-d7f1-cacd-0b5be2f2db9b/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-27T12:23:56,303 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/16bc5e44-ed69-d7f1-cacd-0b5be2f2db9b/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-27T12:23:56,303 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/16bc5e44-ed69-d7f1-cacd-0b5be2f2db9b/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-27T12:23:57,353 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-27T12:23:57,437 INFO [Time-limited test {}] log.Log(170): Logging initialized @2568ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-27T12:23:57,501 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-27T12:23:57,558 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-27T12:23:57,578 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-27T12:23:57,578 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-27T12:23:57,580 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-27T12:23:57,590 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-27T12:23:57,593 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@1efa708{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/16bc5e44-ed69-d7f1-cacd-0b5be2f2db9b/hadoop.log.dir/,AVAILABLE} 2024-11-27T12:23:57,594 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@2e580bc0{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-27T12:23:57,765 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@3f7463d{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/16bc5e44-ed69-d7f1-cacd-0b5be2f2db9b/java.io.tmpdir/jetty-localhost-40985-hadoop-hdfs-3_4_1-tests_jar-_-any-8214915622308783784/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-27T12:23:57,780 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@21e93b68{HTTP/1.1, (http/1.1)}{localhost:40985} 2024-11-27T12:23:57,780 INFO [Time-limited test {}] server.Server(415): Started @2912ms 2024-11-27T12:23:58,352 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-27T12:23:58,360 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-27T12:23:58,361 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-27T12:23:58,361 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-27T12:23:58,362 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-27T12:23:58,362 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@15b00a0c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/16bc5e44-ed69-d7f1-cacd-0b5be2f2db9b/hadoop.log.dir/,AVAILABLE} 2024-11-27T12:23:58,363 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3ad6a89f{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-27T12:23:58,462 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@70061d7e{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/16bc5e44-ed69-d7f1-cacd-0b5be2f2db9b/java.io.tmpdir/jetty-localhost-32987-hadoop-hdfs-3_4_1-tests_jar-_-any-2762042536002846384/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-27T12:23:58,463 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@70195653{HTTP/1.1, (http/1.1)}{localhost:32987} 2024-11-27T12:23:58,463 INFO [Time-limited test {}] server.Server(415): Started @3595ms 2024-11-27T12:23:58,511 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-27T12:23:59,873 WARN [Thread-72 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/16bc5e44-ed69-d7f1-cacd-0b5be2f2db9b/cluster_2674ffaa-0f8a-9abb-ffed-bcc8bba3b369/data/data1/current/BP-588666114-172.17.0.3-1732710236850/current, will proceed with Du for space computation calculation, 2024-11-27T12:23:59,873 WARN [Thread-73 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/16bc5e44-ed69-d7f1-cacd-0b5be2f2db9b/cluster_2674ffaa-0f8a-9abb-ffed-bcc8bba3b369/data/data2/current/BP-588666114-172.17.0.3-1732710236850/current, will proceed with Du for space computation calculation, 2024-11-27T12:23:59,905 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-27T12:23:59,946 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x690b7d3b7a2bda84 with lease ID 0xa9ce447b41423123: Processing first storage report for DS-d207534c-4acd-4501-95b5-4ca3fb7f2487 from datanode DatanodeRegistration(127.0.0.1:37513, datanodeUuid=7c1b600a-6ba2-44c0-98d8-e28cb85bd5bf, infoPort=43533, infoSecurePort=0, ipcPort=44087, storageInfo=lv=-57;cid=testClusterID;nsid=2039447842;c=1732710236850) 2024-11-27T12:23:59,948 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x690b7d3b7a2bda84 with lease ID 0xa9ce447b41423123: from storage DS-d207534c-4acd-4501-95b5-4ca3fb7f2487 node DatanodeRegistration(127.0.0.1:37513, datanodeUuid=7c1b600a-6ba2-44c0-98d8-e28cb85bd5bf, infoPort=43533, infoSecurePort=0, ipcPort=44087, storageInfo=lv=-57;cid=testClusterID;nsid=2039447842;c=1732710236850), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-27T12:23:59,949 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x690b7d3b7a2bda84 with lease ID 0xa9ce447b41423123: Processing first storage report for DS-2185eb47-ebbb-4dfa-85fe-82ab78bf884a from datanode DatanodeRegistration(127.0.0.1:37513, datanodeUuid=7c1b600a-6ba2-44c0-98d8-e28cb85bd5bf, infoPort=43533, infoSecurePort=0, ipcPort=44087, storageInfo=lv=-57;cid=testClusterID;nsid=2039447842;c=1732710236850) 2024-11-27T12:23:59,949 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x690b7d3b7a2bda84 with lease ID 0xa9ce447b41423123: from storage DS-2185eb47-ebbb-4dfa-85fe-82ab78bf884a node DatanodeRegistration(127.0.0.1:37513, datanodeUuid=7c1b600a-6ba2-44c0-98d8-e28cb85bd5bf, infoPort=43533, infoSecurePort=0, ipcPort=44087, storageInfo=lv=-57;cid=testClusterID;nsid=2039447842;c=1732710236850), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-27T12:24:00,016 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/16bc5e44-ed69-d7f1-cacd-0b5be2f2db9b 2024-11-27T12:24:00,077 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/16bc5e44-ed69-d7f1-cacd-0b5be2f2db9b/cluster_2674ffaa-0f8a-9abb-ffed-bcc8bba3b369/zookeeper_0, clientPort=62785, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/16bc5e44-ed69-d7f1-cacd-0b5be2f2db9b/cluster_2674ffaa-0f8a-9abb-ffed-bcc8bba3b369/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/16bc5e44-ed69-d7f1-cacd-0b5be2f2db9b/cluster_2674ffaa-0f8a-9abb-ffed-bcc8bba3b369/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-27T12:24:00,085 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=62785 2024-11-27T12:24:00,100 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-27T12:24:00,103 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-27T12:24:00,299 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073741825_1001 (size=7) 2024-11-27T12:24:00,706 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c with version=8 2024-11-27T12:24:00,706 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/hbase-staging 2024-11-27T12:24:00,787 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-27T12:24:01,009 INFO [Time-limited test {}] client.ConnectionUtils(128): master/1ba5af43fd99:0 server-side Connection retries=45 2024-11-27T12:24:01,017 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-27T12:24:01,018 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-27T12:24:01,022 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-27T12:24:01,022 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-27T12:24:01,023 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-27T12:24:01,146 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-27T12:24:01,200 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-27T12:24:01,209 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-27T12:24:01,213 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-27T12:24:01,236 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 32444 (auto-detected) 2024-11-27T12:24:01,237 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:03 (auto-detected) 2024-11-27T12:24:01,259 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:36215 2024-11-27T12:24:01,288 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:36215 connecting to ZooKeeper ensemble=127.0.0.1:62785 2024-11-27T12:24:01,429 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:362150x0, quorum=127.0.0.1:62785, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-27T12:24:01,433 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:36215-0x1017c400ae90000 connected 2024-11-27T12:24:01,513 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-27T12:24:01,517 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-27T12:24:01,529 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:36215-0x1017c400ae90000, quorum=127.0.0.1:62785, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-27T12:24:01,532 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c, hbase.cluster.distributed=false 2024-11-27T12:24:01,560 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:36215-0x1017c400ae90000, quorum=127.0.0.1:62785, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-27T12:24:01,565 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=36215 2024-11-27T12:24:01,565 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=36215 2024-11-27T12:24:01,566 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=36215 2024-11-27T12:24:01,566 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=36215 2024-11-27T12:24:01,566 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=36215 2024-11-27T12:24:01,661 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/1ba5af43fd99:0 server-side Connection retries=45 2024-11-27T12:24:01,662 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-27T12:24:01,662 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-27T12:24:01,662 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-27T12:24:01,663 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-27T12:24:01,663 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-27T12:24:01,665 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-27T12:24:01,667 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-27T12:24:01,668 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:36251 2024-11-27T12:24:01,670 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:36251 connecting to ZooKeeper ensemble=127.0.0.1:62785 2024-11-27T12:24:01,671 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-27T12:24:01,674 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-27T12:24:01,738 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:362510x0, quorum=127.0.0.1:62785, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-27T12:24:01,739 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:36251-0x1017c400ae90001 connected 2024-11-27T12:24:01,740 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:36251-0x1017c400ae90001, quorum=127.0.0.1:62785, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-27T12:24:01,745 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-27T12:24:01,753 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-27T12:24:01,756 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:36251-0x1017c400ae90001, quorum=127.0.0.1:62785, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-27T12:24:01,760 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:36251-0x1017c400ae90001, quorum=127.0.0.1:62785, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-27T12:24:01,760 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=36251 2024-11-27T12:24:01,761 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=36251 2024-11-27T12:24:01,761 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=36251 2024-11-27T12:24:01,762 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=36251 2024-11-27T12:24:01,762 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=36251 2024-11-27T12:24:01,777 DEBUG [M:0;1ba5af43fd99:36215 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;1ba5af43fd99:36215 2024-11-27T12:24:01,778 INFO [master/1ba5af43fd99:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/1ba5af43fd99,36215,1732710240860 2024-11-27T12:24:01,825 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36251-0x1017c400ae90001, quorum=127.0.0.1:62785, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-27T12:24:01,825 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36215-0x1017c400ae90000, quorum=127.0.0.1:62785, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-27T12:24:01,827 DEBUG [master/1ba5af43fd99:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:36215-0x1017c400ae90000, quorum=127.0.0.1:62785, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/1ba5af43fd99,36215,1732710240860 2024-11-27T12:24:01,909 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36215-0x1017c400ae90000, quorum=127.0.0.1:62785, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T12:24:01,909 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36251-0x1017c400ae90001, quorum=127.0.0.1:62785, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-27T12:24:01,909 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36251-0x1017c400ae90001, quorum=127.0.0.1:62785, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T12:24:01,910 DEBUG [master/1ba5af43fd99:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:36215-0x1017c400ae90000, quorum=127.0.0.1:62785, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-27T12:24:01,911 INFO [master/1ba5af43fd99:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/1ba5af43fd99,36215,1732710240860 from backup master directory 2024-11-27T12:24:01,970 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36251-0x1017c400ae90001, quorum=127.0.0.1:62785, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-27T12:24:01,970 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36215-0x1017c400ae90000, quorum=127.0.0.1:62785, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/1ba5af43fd99,36215,1732710240860 2024-11-27T12:24:01,970 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36215-0x1017c400ae90000, quorum=127.0.0.1:62785, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-27T12:24:01,971 WARN [master/1ba5af43fd99:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-27T12:24:01,971 INFO [master/1ba5af43fd99:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=1ba5af43fd99,36215,1732710240860 2024-11-27T12:24:01,973 INFO [master/1ba5af43fd99:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-27T12:24:01,975 INFO [master/1ba5af43fd99:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-27T12:24:02,026 DEBUG [master/1ba5af43fd99:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/hbase.id] with ID: 200978c2-2c41-4f93-b0b5-127ea1024f37 2024-11-27T12:24:02,027 DEBUG [master/1ba5af43fd99:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/.tmp/hbase.id 2024-11-27T12:24:02,036 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073741826_1002 (size=42) 2024-11-27T12:24:02,442 DEBUG [master/1ba5af43fd99:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/.tmp/hbase.id]:[hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/hbase.id] 2024-11-27T12:24:02,496 INFO [master/1ba5af43fd99:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-27T12:24:02,501 INFO [master/1ba5af43fd99:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-27T12:24:02,520 INFO [master/1ba5af43fd99:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 17ms. 2024-11-27T12:24:02,573 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36215-0x1017c400ae90000, quorum=127.0.0.1:62785, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T12:24:02,573 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36251-0x1017c400ae90001, quorum=127.0.0.1:62785, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T12:24:02,591 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073741827_1003 (size=196) 2024-11-27T12:24:02,609 INFO [master/1ba5af43fd99:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-27T12:24:02,612 INFO [master/1ba5af43fd99:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-27T12:24:02,625 DEBUG [master/1ba5af43fd99:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:24:02,628 INFO [master/1ba5af43fd99:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-27T12:24:02,655 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073741828_1004 (size=1189) 2024-11-27T12:24:03,078 INFO [master/1ba5af43fd99:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/MasterData/data/master/store 2024-11-27T12:24:03,095 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073741829_1005 (size=34) 2024-11-27T12:24:03,506 INFO [master/1ba5af43fd99:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-27T12:24:03,509 DEBUG [master/1ba5af43fd99:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T12:24:03,510 DEBUG [master/1ba5af43fd99:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-27T12:24:03,510 INFO [master/1ba5af43fd99:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-27T12:24:03,510 DEBUG [master/1ba5af43fd99:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-27T12:24:03,511 DEBUG [master/1ba5af43fd99:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-27T12:24:03,511 DEBUG [master/1ba5af43fd99:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-27T12:24:03,512 INFO [master/1ba5af43fd99:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-27T12:24:03,512 DEBUG [master/1ba5af43fd99:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732710243510Disabling compacts and flushes for region at 1732710243510Disabling writes for close at 1732710243511 (+1 ms)Writing region close event to WAL at 1732710243511Closed at 1732710243511 2024-11-27T12:24:03,515 WARN [master/1ba5af43fd99:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/MasterData/data/master/store/.initializing 2024-11-27T12:24:03,515 DEBUG [master/1ba5af43fd99:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/MasterData/WALs/1ba5af43fd99,36215,1732710240860 2024-11-27T12:24:03,522 INFO [master/1ba5af43fd99:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-27T12:24:03,535 INFO [master/1ba5af43fd99:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=1ba5af43fd99%2C36215%2C1732710240860, suffix=, logDir=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/MasterData/WALs/1ba5af43fd99,36215,1732710240860, archiveDir=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/MasterData/oldWALs, maxLogs=10 2024-11-27T12:24:03,555 DEBUG [master/1ba5af43fd99:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/MasterData/WALs/1ba5af43fd99,36215,1732710240860/1ba5af43fd99%2C36215%2C1732710240860.1732710243539, exclude list is [], retry=0 2024-11-27T12:24:03,572 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37513,DS-d207534c-4acd-4501-95b5-4ca3fb7f2487,DISK] 2024-11-27T12:24:03,575 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-27T12:24:03,606 INFO [master/1ba5af43fd99:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/MasterData/WALs/1ba5af43fd99,36215,1732710240860/1ba5af43fd99%2C36215%2C1732710240860.1732710243539 2024-11-27T12:24:03,607 DEBUG [master/1ba5af43fd99:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:43533:43533)] 2024-11-27T12:24:03,608 DEBUG [master/1ba5af43fd99:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-27T12:24:03,608 DEBUG [master/1ba5af43fd99:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T12:24:03,611 DEBUG [master/1ba5af43fd99:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-27T12:24:03,612 DEBUG [master/1ba5af43fd99:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-27T12:24:03,654 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-27T12:24:03,687 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-27T12:24:03,692 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:03,695 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-27T12:24:03,696 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-27T12:24:03,700 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-27T12:24:03,700 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:03,701 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T12:24:03,701 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-27T12:24:03,705 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-27T12:24:03,705 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:03,706 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T12:24:03,706 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-27T12:24:03,709 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-27T12:24:03,710 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:03,711 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T12:24:03,711 DEBUG [master/1ba5af43fd99:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-27T12:24:03,715 DEBUG [master/1ba5af43fd99:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-27T12:24:03,716 DEBUG [master/1ba5af43fd99:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-27T12:24:03,722 DEBUG [master/1ba5af43fd99:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-27T12:24:03,723 DEBUG [master/1ba5af43fd99:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-27T12:24:03,728 DEBUG [master/1ba5af43fd99:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-27T12:24:03,733 DEBUG [master/1ba5af43fd99:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-27T12:24:03,739 DEBUG [master/1ba5af43fd99:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-27T12:24:03,741 INFO [master/1ba5af43fd99:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64978424, jitterRate=-0.0317460298538208}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-27T12:24:03,749 DEBUG [master/1ba5af43fd99:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1732710243625Initializing all the Stores at 1732710243627 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732710243628 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732710243629 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732710243629Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732710243629Cleaning up temporary data from old regions at 1732710243723 (+94 ms)Region opened successfully at 1732710243748 (+25 ms) 2024-11-27T12:24:03,750 INFO [master/1ba5af43fd99:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-27T12:24:03,784 DEBUG [master/1ba5af43fd99:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@77cb685d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=1ba5af43fd99/172.17.0.3:0 2024-11-27T12:24:03,812 INFO [master/1ba5af43fd99:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-27T12:24:03,822 INFO [master/1ba5af43fd99:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-27T12:24:03,822 INFO [master/1ba5af43fd99:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-27T12:24:03,825 INFO [master/1ba5af43fd99:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-27T12:24:03,826 INFO [master/1ba5af43fd99:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-27T12:24:03,830 INFO [master/1ba5af43fd99:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 3 msec 2024-11-27T12:24:03,830 INFO [master/1ba5af43fd99:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-27T12:24:03,854 INFO [master/1ba5af43fd99:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-27T12:24:03,864 DEBUG [master/1ba5af43fd99:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36215-0x1017c400ae90000, quorum=127.0.0.1:62785, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-27T12:24:03,888 DEBUG [master/1ba5af43fd99:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-27T12:24:03,891 INFO [master/1ba5af43fd99:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-27T12:24:03,892 DEBUG [master/1ba5af43fd99:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36215-0x1017c400ae90000, quorum=127.0.0.1:62785, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-27T12:24:03,899 DEBUG [master/1ba5af43fd99:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-27T12:24:03,901 INFO [master/1ba5af43fd99:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-27T12:24:03,906 DEBUG [master/1ba5af43fd99:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36215-0x1017c400ae90000, quorum=127.0.0.1:62785, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-27T12:24:03,985 DEBUG [master/1ba5af43fd99:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-27T12:24:03,989 DEBUG [master/1ba5af43fd99:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36215-0x1017c400ae90000, quorum=127.0.0.1:62785, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-27T12:24:04,004 DEBUG [master/1ba5af43fd99:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-27T12:24:04,026 DEBUG [master/1ba5af43fd99:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36215-0x1017c400ae90000, quorum=127.0.0.1:62785, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-27T12:24:04,036 DEBUG [master/1ba5af43fd99:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-27T12:24:04,046 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36215-0x1017c400ae90000, quorum=127.0.0.1:62785, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-27T12:24:04,047 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36215-0x1017c400ae90000, quorum=127.0.0.1:62785, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T12:24:04,047 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36251-0x1017c400ae90001, quorum=127.0.0.1:62785, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-27T12:24:04,047 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36251-0x1017c400ae90001, quorum=127.0.0.1:62785, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T12:24:04,051 INFO [master/1ba5af43fd99:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=1ba5af43fd99,36215,1732710240860, sessionid=0x1017c400ae90000, setting cluster-up flag (Was=false) 2024-11-27T12:24:04,078 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36215-0x1017c400ae90000, quorum=127.0.0.1:62785, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T12:24:04,078 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36251-0x1017c400ae90001, quorum=127.0.0.1:62785, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T12:24:04,109 DEBUG [master/1ba5af43fd99:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-27T12:24:04,112 DEBUG [master/1ba5af43fd99:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=1ba5af43fd99,36215,1732710240860 2024-11-27T12:24:04,209 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36215-0x1017c400ae90000, quorum=127.0.0.1:62785, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T12:24:04,209 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36251-0x1017c400ae90001, quorum=127.0.0.1:62785, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T12:24:04,393 DEBUG [master/1ba5af43fd99:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-27T12:24:04,400 DEBUG [master/1ba5af43fd99:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=1ba5af43fd99,36215,1732710240860 2024-11-27T12:24:04,408 INFO [master/1ba5af43fd99:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-27T12:24:04,469 INFO [RS:0;1ba5af43fd99:36251 {}] regionserver.HRegionServer(746): ClusterId : 200978c2-2c41-4f93-b0b5-127ea1024f37 2024-11-27T12:24:04,471 DEBUG [RS:0;1ba5af43fd99:36251 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-27T12:24:04,472 DEBUG [master/1ba5af43fd99:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-27T12:24:04,481 INFO [master/1ba5af43fd99:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-27T12:24:04,486 INFO [master/1ba5af43fd99:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-27T12:24:04,491 DEBUG [master/1ba5af43fd99:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 1ba5af43fd99,36215,1732710240860 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-27T12:24:04,501 DEBUG [RS:0;1ba5af43fd99:36251 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-27T12:24:04,501 DEBUG [RS:0;1ba5af43fd99:36251 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-27T12:24:04,501 DEBUG [master/1ba5af43fd99:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/1ba5af43fd99:0, corePoolSize=5, maxPoolSize=5 2024-11-27T12:24:04,501 DEBUG [master/1ba5af43fd99:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/1ba5af43fd99:0, corePoolSize=5, maxPoolSize=5 2024-11-27T12:24:04,502 DEBUG [master/1ba5af43fd99:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/1ba5af43fd99:0, corePoolSize=5, maxPoolSize=5 2024-11-27T12:24:04,502 DEBUG [master/1ba5af43fd99:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/1ba5af43fd99:0, corePoolSize=5, maxPoolSize=5 2024-11-27T12:24:04,502 DEBUG [master/1ba5af43fd99:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/1ba5af43fd99:0, corePoolSize=10, maxPoolSize=10 2024-11-27T12:24:04,502 DEBUG [master/1ba5af43fd99:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/1ba5af43fd99:0, corePoolSize=1, maxPoolSize=1 2024-11-27T12:24:04,502 DEBUG [master/1ba5af43fd99:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/1ba5af43fd99:0, corePoolSize=2, maxPoolSize=2 2024-11-27T12:24:04,503 DEBUG [master/1ba5af43fd99:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/1ba5af43fd99:0, corePoolSize=1, maxPoolSize=1 2024-11-27T12:24:04,504 INFO [master/1ba5af43fd99:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732710274504 2024-11-27T12:24:04,505 INFO [master/1ba5af43fd99:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-27T12:24:04,506 INFO [master/1ba5af43fd99:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-27T12:24:04,508 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-27T12:24:04,509 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-27T12:24:04,509 INFO [master/1ba5af43fd99:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-27T12:24:04,510 INFO [master/1ba5af43fd99:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-27T12:24:04,510 INFO [master/1ba5af43fd99:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-27T12:24:04,511 INFO [master/1ba5af43fd99:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-27T12:24:04,511 DEBUG [RS:0;1ba5af43fd99:36251 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-27T12:24:04,512 DEBUG [RS:0;1ba5af43fd99:36251 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5c652a3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=1ba5af43fd99/172.17.0.3:0 2024-11-27T12:24:04,512 INFO [master/1ba5af43fd99:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-27T12:24:04,515 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:04,515 INFO [master/1ba5af43fd99:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-27T12:24:04,515 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-27T12:24:04,516 INFO [master/1ba5af43fd99:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-27T12:24:04,517 INFO [master/1ba5af43fd99:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-27T12:24:04,519 INFO [master/1ba5af43fd99:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-27T12:24:04,519 INFO [master/1ba5af43fd99:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-27T12:24:04,522 DEBUG [master/1ba5af43fd99:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/1ba5af43fd99:0:becomeActiveMaster-HFileCleaner.large.0-1732710244521,5,FailOnTimeoutGroup] 2024-11-27T12:24:04,525 DEBUG [master/1ba5af43fd99:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/1ba5af43fd99:0:becomeActiveMaster-HFileCleaner.small.0-1732710244522,5,FailOnTimeoutGroup] 2024-11-27T12:24:04,525 INFO [master/1ba5af43fd99:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-27T12:24:04,525 INFO [master/1ba5af43fd99:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-27T12:24:04,527 INFO [master/1ba5af43fd99:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-27T12:24:04,527 DEBUG [RS:0;1ba5af43fd99:36251 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;1ba5af43fd99:36251 2024-11-27T12:24:04,527 INFO [master/1ba5af43fd99:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-27T12:24:04,527 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073741831_1007 (size=1321) 2024-11-27T12:24:04,530 INFO [RS:0;1ba5af43fd99:36251 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-27T12:24:04,530 INFO [RS:0;1ba5af43fd99:36251 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-27T12:24:04,530 DEBUG [RS:0;1ba5af43fd99:36251 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-27T12:24:04,533 INFO [RS:0;1ba5af43fd99:36251 {}] regionserver.HRegionServer(2659): reportForDuty to master=1ba5af43fd99,36215,1732710240860 with port=36251, startcode=1732710241631 2024-11-27T12:24:04,543 DEBUG [RS:0;1ba5af43fd99:36251 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-27T12:24:04,573 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47253, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-27T12:24:04,579 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36215 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:04,580 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36215 {}] master.ServerManager(517): Registering regionserver=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:04,592 DEBUG [RS:0;1ba5af43fd99:36251 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c 2024-11-27T12:24:04,593 DEBUG [RS:0;1ba5af43fd99:36251 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:45125 2024-11-27T12:24:04,593 DEBUG [RS:0;1ba5af43fd99:36251 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-27T12:24:04,604 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36215-0x1017c400ae90000, quorum=127.0.0.1:62785, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-27T12:24:04,606 DEBUG [RS:0;1ba5af43fd99:36251 {}] zookeeper.ZKUtil(111): regionserver:36251-0x1017c400ae90001, quorum=127.0.0.1:62785, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:04,606 WARN [RS:0;1ba5af43fd99:36251 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-27T12:24:04,606 INFO [RS:0;1ba5af43fd99:36251 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-27T12:24:04,606 DEBUG [RS:0;1ba5af43fd99:36251 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/WALs/1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:04,608 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [1ba5af43fd99,36251,1732710241631] 2024-11-27T12:24:04,630 INFO [RS:0;1ba5af43fd99:36251 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-27T12:24:04,645 INFO [RS:0;1ba5af43fd99:36251 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-27T12:24:04,650 INFO [RS:0;1ba5af43fd99:36251 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-27T12:24:04,650 INFO [RS:0;1ba5af43fd99:36251 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-27T12:24:04,651 INFO [RS:0;1ba5af43fd99:36251 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-27T12:24:04,656 INFO [RS:0;1ba5af43fd99:36251 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-27T12:24:04,657 INFO [RS:0;1ba5af43fd99:36251 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-27T12:24:04,658 DEBUG [RS:0;1ba5af43fd99:36251 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/1ba5af43fd99:0, corePoolSize=1, maxPoolSize=1 2024-11-27T12:24:04,658 DEBUG [RS:0;1ba5af43fd99:36251 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/1ba5af43fd99:0, corePoolSize=1, maxPoolSize=1 2024-11-27T12:24:04,658 DEBUG [RS:0;1ba5af43fd99:36251 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/1ba5af43fd99:0, corePoolSize=1, maxPoolSize=1 2024-11-27T12:24:04,658 DEBUG [RS:0;1ba5af43fd99:36251 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/1ba5af43fd99:0, corePoolSize=1, maxPoolSize=1 2024-11-27T12:24:04,658 DEBUG [RS:0;1ba5af43fd99:36251 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/1ba5af43fd99:0, corePoolSize=1, maxPoolSize=1 2024-11-27T12:24:04,658 DEBUG [RS:0;1ba5af43fd99:36251 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/1ba5af43fd99:0, corePoolSize=2, maxPoolSize=2 2024-11-27T12:24:04,659 DEBUG [RS:0;1ba5af43fd99:36251 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/1ba5af43fd99:0, corePoolSize=1, maxPoolSize=1 2024-11-27T12:24:04,659 DEBUG [RS:0;1ba5af43fd99:36251 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/1ba5af43fd99:0, corePoolSize=1, maxPoolSize=1 2024-11-27T12:24:04,659 DEBUG [RS:0;1ba5af43fd99:36251 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/1ba5af43fd99:0, corePoolSize=1, maxPoolSize=1 2024-11-27T12:24:04,659 DEBUG [RS:0;1ba5af43fd99:36251 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/1ba5af43fd99:0, corePoolSize=1, maxPoolSize=1 2024-11-27T12:24:04,659 DEBUG [RS:0;1ba5af43fd99:36251 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/1ba5af43fd99:0, corePoolSize=1, maxPoolSize=1 2024-11-27T12:24:04,659 DEBUG [RS:0;1ba5af43fd99:36251 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/1ba5af43fd99:0, corePoolSize=1, maxPoolSize=1 2024-11-27T12:24:04,660 DEBUG [RS:0;1ba5af43fd99:36251 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/1ba5af43fd99:0, corePoolSize=3, maxPoolSize=3 2024-11-27T12:24:04,660 DEBUG [RS:0;1ba5af43fd99:36251 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0, corePoolSize=3, maxPoolSize=3 2024-11-27T12:24:04,661 INFO [RS:0;1ba5af43fd99:36251 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-27T12:24:04,661 INFO [RS:0;1ba5af43fd99:36251 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-27T12:24:04,661 INFO [RS:0;1ba5af43fd99:36251 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-27T12:24:04,661 INFO [RS:0;1ba5af43fd99:36251 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-27T12:24:04,661 INFO [RS:0;1ba5af43fd99:36251 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-27T12:24:04,661 INFO [RS:0;1ba5af43fd99:36251 {}] hbase.ChoreService(168): Chore ScheduledChore name=1ba5af43fd99,36251,1732710241631-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-27T12:24:04,681 INFO [RS:0;1ba5af43fd99:36251 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-27T12:24:04,683 INFO [RS:0;1ba5af43fd99:36251 {}] hbase.ChoreService(168): Chore ScheduledChore name=1ba5af43fd99,36251,1732710241631-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-27T12:24:04,684 INFO [RS:0;1ba5af43fd99:36251 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-27T12:24:04,684 INFO [RS:0;1ba5af43fd99:36251 {}] regionserver.Replication(171): 1ba5af43fd99,36251,1732710241631 started 2024-11-27T12:24:04,706 INFO [RS:0;1ba5af43fd99:36251 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-27T12:24:04,707 INFO [RS:0;1ba5af43fd99:36251 {}] regionserver.HRegionServer(1482): Serving as 1ba5af43fd99,36251,1732710241631, RpcServer on 1ba5af43fd99/172.17.0.3:36251, sessionid=0x1017c400ae90001 2024-11-27T12:24:04,708 DEBUG [RS:0;1ba5af43fd99:36251 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-27T12:24:04,708 DEBUG [RS:0;1ba5af43fd99:36251 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:04,708 DEBUG [RS:0;1ba5af43fd99:36251 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '1ba5af43fd99,36251,1732710241631' 2024-11-27T12:24:04,708 DEBUG [RS:0;1ba5af43fd99:36251 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-27T12:24:04,709 DEBUG [RS:0;1ba5af43fd99:36251 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-27T12:24:04,710 DEBUG [RS:0;1ba5af43fd99:36251 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-27T12:24:04,710 DEBUG [RS:0;1ba5af43fd99:36251 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-27T12:24:04,711 DEBUG [RS:0;1ba5af43fd99:36251 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:04,711 DEBUG [RS:0;1ba5af43fd99:36251 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '1ba5af43fd99,36251,1732710241631' 2024-11-27T12:24:04,711 DEBUG [RS:0;1ba5af43fd99:36251 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-27T12:24:04,712 DEBUG [RS:0;1ba5af43fd99:36251 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-27T12:24:04,712 DEBUG [RS:0;1ba5af43fd99:36251 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-27T12:24:04,712 INFO [RS:0;1ba5af43fd99:36251 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-27T12:24:04,712 INFO [RS:0;1ba5af43fd99:36251 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-27T12:24:04,820 INFO [RS:0;1ba5af43fd99:36251 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-27T12:24:04,824 INFO [RS:0;1ba5af43fd99:36251 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=1ba5af43fd99%2C36251%2C1732710241631, suffix=, logDir=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/WALs/1ba5af43fd99,36251,1732710241631, archiveDir=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/oldWALs, maxLogs=32 2024-11-27T12:24:04,840 DEBUG [RS:0;1ba5af43fd99:36251 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/WALs/1ba5af43fd99,36251,1732710241631/1ba5af43fd99%2C36251%2C1732710241631.1732710244827, exclude list is [], retry=0 2024-11-27T12:24:04,845 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37513,DS-d207534c-4acd-4501-95b5-4ca3fb7f2487,DISK] 2024-11-27T12:24:04,848 INFO [RS:0;1ba5af43fd99:36251 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/WALs/1ba5af43fd99,36251,1732710241631/1ba5af43fd99%2C36251%2C1732710241631.1732710244827 2024-11-27T12:24:04,849 DEBUG [RS:0;1ba5af43fd99:36251 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:43533:43533)] 2024-11-27T12:24:04,930 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-27T12:24:04,931 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c 2024-11-27T12:24:04,940 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073741833_1009 (size=32) 2024-11-27T12:24:05,343 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T12:24:05,346 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-27T12:24:05,349 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-27T12:24:05,349 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:05,350 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-27T12:24:05,350 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-27T12:24:05,353 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-27T12:24:05,353 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:05,354 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-27T12:24:05,354 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-27T12:24:05,357 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-27T12:24:05,357 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:05,359 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-27T12:24:05,359 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-27T12:24:05,362 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-27T12:24:05,363 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:05,364 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-27T12:24:05,364 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-27T12:24:05,366 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/hbase/meta/1588230740 2024-11-27T12:24:05,367 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/hbase/meta/1588230740 2024-11-27T12:24:05,371 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-27T12:24:05,371 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-27T12:24:05,372 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-27T12:24:05,376 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-27T12:24:05,380 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-27T12:24:05,381 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63701822, jitterRate=-0.050768882036209106}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-27T12:24:05,384 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1732710245344Initializing all the Stores at 1732710245346 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732710245346Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732710245346Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732710245346Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732710245346Cleaning up temporary data from old regions at 1732710245371 (+25 ms)Region opened successfully at 1732710245384 (+13 ms) 2024-11-27T12:24:05,384 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-27T12:24:05,384 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-27T12:24:05,384 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-27T12:24:05,384 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-27T12:24:05,384 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-27T12:24:05,386 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-27T12:24:05,386 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732710245384Disabling compacts and flushes for region at 1732710245384Disabling writes for close at 1732710245384Writing region close event to WAL at 1732710245385 (+1 ms)Closed at 1732710245386 (+1 ms) 2024-11-27T12:24:05,389 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-27T12:24:05,389 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-27T12:24:05,395 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-27T12:24:05,402 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-27T12:24:05,405 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-27T12:24:05,558 DEBUG [1ba5af43fd99:36215 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-27T12:24:05,568 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:05,573 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 1ba5af43fd99,36251,1732710241631, state=OPENING 2024-11-27T12:24:05,594 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-27T12:24:05,604 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36251-0x1017c400ae90001, quorum=127.0.0.1:62785, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T12:24:05,604 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36215-0x1017c400ae90000, quorum=127.0.0.1:62785, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T12:24:05,605 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-27T12:24:05,605 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-27T12:24:05,607 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-27T12:24:05,609 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=1ba5af43fd99,36251,1732710241631}] 2024-11-27T12:24:05,789 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-27T12:24:05,793 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45163, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-27T12:24:05,804 INFO [RS_OPEN_META-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-27T12:24:05,804 INFO [RS_OPEN_META-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-27T12:24:05,805 INFO [RS_OPEN_META-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-27T12:24:05,809 INFO [RS_OPEN_META-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=1ba5af43fd99%2C36251%2C1732710241631.meta, suffix=.meta, logDir=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/WALs/1ba5af43fd99,36251,1732710241631, archiveDir=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/oldWALs, maxLogs=32 2024-11-27T12:24:05,827 DEBUG [RS_OPEN_META-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/WALs/1ba5af43fd99,36251,1732710241631/1ba5af43fd99%2C36251%2C1732710241631.meta.1732710245812.meta, exclude list is [], retry=0 2024-11-27T12:24:05,831 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37513,DS-d207534c-4acd-4501-95b5-4ca3fb7f2487,DISK] 2024-11-27T12:24:05,834 INFO [RS_OPEN_META-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/WALs/1ba5af43fd99,36251,1732710241631/1ba5af43fd99%2C36251%2C1732710241631.meta.1732710245812.meta 2024-11-27T12:24:05,835 DEBUG [RS_OPEN_META-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:43533:43533)] 2024-11-27T12:24:05,835 DEBUG [RS_OPEN_META-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-27T12:24:05,837 DEBUG [RS_OPEN_META-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-27T12:24:05,839 DEBUG [RS_OPEN_META-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-27T12:24:05,843 INFO [RS_OPEN_META-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-27T12:24:05,847 DEBUG [RS_OPEN_META-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-27T12:24:05,848 DEBUG [RS_OPEN_META-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T12:24:05,848 DEBUG [RS_OPEN_META-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-27T12:24:05,848 DEBUG [RS_OPEN_META-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-27T12:24:05,851 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-27T12:24:05,853 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-27T12:24:05,853 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:05,854 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-27T12:24:05,854 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-27T12:24:05,856 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-27T12:24:05,856 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:05,857 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-27T12:24:05,857 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-27T12:24:05,859 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-27T12:24:05,859 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:05,860 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-27T12:24:05,860 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-27T12:24:05,861 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-27T12:24:05,861 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:05,862 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-27T12:24:05,863 DEBUG [RS_OPEN_META-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-27T12:24:05,864 DEBUG [RS_OPEN_META-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/hbase/meta/1588230740 2024-11-27T12:24:05,867 DEBUG [RS_OPEN_META-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/hbase/meta/1588230740 2024-11-27T12:24:05,869 DEBUG [RS_OPEN_META-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-27T12:24:05,869 DEBUG [RS_OPEN_META-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-27T12:24:05,870 DEBUG [RS_OPEN_META-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-27T12:24:05,873 DEBUG [RS_OPEN_META-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-27T12:24:05,875 INFO [RS_OPEN_META-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71519309, jitterRate=0.06572075188159943}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-27T12:24:05,875 DEBUG [RS_OPEN_META-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-27T12:24:05,876 DEBUG [RS_OPEN_META-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1732710245849Writing region info on filesystem at 1732710245849Initializing all the Stores at 1732710245851 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732710245851Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732710245851Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732710245851Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732710245851Cleaning up temporary data from old regions at 1732710245869 (+18 ms)Running coprocessor post-open hooks at 1732710245875 (+6 ms)Region opened successfully at 1732710245876 (+1 ms) 2024-11-27T12:24:05,883 INFO [RS_OPEN_META-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1732710245778 2024-11-27T12:24:05,897 DEBUG [RS_OPEN_META-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-27T12:24:05,898 INFO [RS_OPEN_META-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-27T12:24:05,898 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:05,901 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 1ba5af43fd99,36251,1732710241631, state=OPEN 2024-11-27T12:24:06,023 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36215-0x1017c400ae90000, quorum=127.0.0.1:62785, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-27T12:24:06,023 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36251-0x1017c400ae90001, quorum=127.0.0.1:62785, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-27T12:24:06,023 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-27T12:24:06,023 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-27T12:24:06,023 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:06,028 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-27T12:24:06,029 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=1ba5af43fd99,36251,1732710241631 in 414 msec 2024-11-27T12:24:06,035 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-27T12:24:06,035 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 636 msec 2024-11-27T12:24:06,037 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-27T12:24:06,037 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-27T12:24:06,055 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T12:24:06,056 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1ba5af43fd99,36251,1732710241631, seqNum=-1] 2024-11-27T12:24:06,074 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T12:24:06,076 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55373, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T12:24:06,096 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.6590 sec 2024-11-27T12:24:06,097 INFO [master/1ba5af43fd99:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1732710246097, completionTime=-1 2024-11-27T12:24:06,100 INFO [master/1ba5af43fd99:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-11-27T12:24:06,100 DEBUG [master/1ba5af43fd99:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-27T12:24:06,125 INFO [master/1ba5af43fd99:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-11-27T12:24:06,125 INFO [master/1ba5af43fd99:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732710306125 2024-11-27T12:24:06,125 INFO [master/1ba5af43fd99:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732710366125 2024-11-27T12:24:06,125 INFO [master/1ba5af43fd99:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 25 msec 2024-11-27T12:24:06,127 INFO [master/1ba5af43fd99:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=1ba5af43fd99,36215,1732710240860-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-27T12:24:06,127 INFO [master/1ba5af43fd99:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=1ba5af43fd99,36215,1732710240860-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-27T12:24:06,127 INFO [master/1ba5af43fd99:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=1ba5af43fd99,36215,1732710240860-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-27T12:24:06,129 INFO [master/1ba5af43fd99:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-1ba5af43fd99:36215, period=300000, unit=MILLISECONDS is enabled. 2024-11-27T12:24:06,129 INFO [master/1ba5af43fd99:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-27T12:24:06,130 INFO [master/1ba5af43fd99:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-27T12:24:06,135 DEBUG [master/1ba5af43fd99:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-27T12:24:06,158 INFO [master/1ba5af43fd99:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 4.187sec 2024-11-27T12:24:06,159 INFO [master/1ba5af43fd99:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-27T12:24:06,160 INFO [master/1ba5af43fd99:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-27T12:24:06,161 INFO [master/1ba5af43fd99:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-27T12:24:06,161 INFO [master/1ba5af43fd99:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-27T12:24:06,161 INFO [master/1ba5af43fd99:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-27T12:24:06,162 INFO [master/1ba5af43fd99:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=1ba5af43fd99,36215,1732710240860-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-27T12:24:06,162 INFO [master/1ba5af43fd99:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=1ba5af43fd99,36215,1732710240860-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-27T12:24:06,170 DEBUG [master/1ba5af43fd99:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-27T12:24:06,171 INFO [master/1ba5af43fd99:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-27T12:24:06,171 INFO [master/1ba5af43fd99:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=1ba5af43fd99,36215,1732710240860-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-27T12:24:06,179 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3d9654c1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T12:24:06,181 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-27T12:24:06,181 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-27T12:24:06,184 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1ba5af43fd99,36215,-1 for getting cluster id 2024-11-27T12:24:06,187 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T12:24:06,194 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '200978c2-2c41-4f93-b0b5-127ea1024f37' 2024-11-27T12:24:06,196 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T12:24:06,196 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "200978c2-2c41-4f93-b0b5-127ea1024f37" 2024-11-27T12:24:06,197 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5f86640e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T12:24:06,197 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1ba5af43fd99,36215,-1] 2024-11-27T12:24:06,199 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T12:24:06,200 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T12:24:06,202 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56388, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T12:24:06,204 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5c7449b9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T12:24:06,205 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T12:24:06,211 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1ba5af43fd99,36251,1732710241631, seqNum=-1] 2024-11-27T12:24:06,212 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T12:24:06,214 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60272, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T12:24:06,235 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=1ba5af43fd99,36215,1732710240860 2024-11-27T12:24:06,246 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMixedAtomicity Thread=211, OpenFileDescriptor=441, MaxFileDescriptor=1048576, SystemLoadAverage=386, ProcessCount=11, AvailableMemoryMB=9896 2024-11-27T12:24:06,254 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-27T12:24:06,259 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 1ba5af43fd99,36215,1732710240860 2024-11-27T12:24:06,277 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@713ddca7 2024-11-27T12:24:06,279 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-27T12:24:06,283 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56394, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-27T12:24:06,286 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-27T12:24:06,291 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-27T12:24:06,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-27T12:24:06,309 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-27T12:24:06,312 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 4 2024-11-27T12:24:06,312 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:06,316 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-27T12:24:06,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-27T12:24:06,328 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073741835_1011 (size=959) 2024-11-27T12:24:06,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-27T12:24:06,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-27T12:24:06,734 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 953f259f29896493f17802c953dd7fd4, NAME => 'TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c 2024-11-27T12:24:06,746 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073741836_1012 (size=53) 2024-11-27T12:24:06,747 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T12:24:06,748 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 953f259f29896493f17802c953dd7fd4, disabling compactions & flushes 2024-11-27T12:24:06,748 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4. 2024-11-27T12:24:06,748 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4. 2024-11-27T12:24:06,748 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4. after waiting 0 ms 2024-11-27T12:24:06,748 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4. 2024-11-27T12:24:06,748 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4. 2024-11-27T12:24:06,748 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 953f259f29896493f17802c953dd7fd4: Waiting for close lock at 1732710246747Disabling compacts and flushes for region at 1732710246747Disabling writes for close at 1732710246748 (+1 ms)Writing region close event to WAL at 1732710246748Closed at 1732710246748 2024-11-27T12:24:06,751 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-27T12:24:06,756 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732710246751"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732710246751"}]},"ts":"1732710246751"} 2024-11-27T12:24:06,760 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-27T12:24:06,762 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-27T12:24:06,765 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732710246763"}]},"ts":"1732710246763"} 2024-11-27T12:24:06,769 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-27T12:24:06,770 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=953f259f29896493f17802c953dd7fd4, ASSIGN}] 2024-11-27T12:24:06,773 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=953f259f29896493f17802c953dd7fd4, ASSIGN 2024-11-27T12:24:06,774 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=953f259f29896493f17802c953dd7fd4, ASSIGN; state=OFFLINE, location=1ba5af43fd99,36251,1732710241631; forceNewPlan=false, retain=false 2024-11-27T12:24:06,928 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=953f259f29896493f17802c953dd7fd4, regionState=OPENING, regionLocation=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:06,938 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=953f259f29896493f17802c953dd7fd4, ASSIGN because future has completed 2024-11-27T12:24:06,940 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631}] 2024-11-27T12:24:06,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-27T12:24:07,105 INFO [RS_OPEN_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4. 2024-11-27T12:24:07,106 DEBUG [RS_OPEN_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 953f259f29896493f17802c953dd7fd4, NAME => 'TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4.', STARTKEY => '', ENDKEY => ''} 2024-11-27T12:24:07,107 DEBUG [RS_OPEN_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 953f259f29896493f17802c953dd7fd4 2024-11-27T12:24:07,107 DEBUG [RS_OPEN_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T12:24:07,107 DEBUG [RS_OPEN_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 953f259f29896493f17802c953dd7fd4 2024-11-27T12:24:07,107 DEBUG [RS_OPEN_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 953f259f29896493f17802c953dd7fd4 2024-11-27T12:24:07,111 INFO [StoreOpener-953f259f29896493f17802c953dd7fd4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 953f259f29896493f17802c953dd7fd4 2024-11-27T12:24:07,114 INFO [StoreOpener-953f259f29896493f17802c953dd7fd4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 953f259f29896493f17802c953dd7fd4 columnFamilyName A 2024-11-27T12:24:07,114 DEBUG [StoreOpener-953f259f29896493f17802c953dd7fd4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:07,115 INFO [StoreOpener-953f259f29896493f17802c953dd7fd4-1 {}] regionserver.HStore(327): Store=953f259f29896493f17802c953dd7fd4/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T12:24:07,115 INFO [StoreOpener-953f259f29896493f17802c953dd7fd4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 953f259f29896493f17802c953dd7fd4 2024-11-27T12:24:07,118 INFO [StoreOpener-953f259f29896493f17802c953dd7fd4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 953f259f29896493f17802c953dd7fd4 columnFamilyName B 2024-11-27T12:24:07,118 DEBUG [StoreOpener-953f259f29896493f17802c953dd7fd4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:07,119 INFO [StoreOpener-953f259f29896493f17802c953dd7fd4-1 {}] regionserver.HStore(327): Store=953f259f29896493f17802c953dd7fd4/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T12:24:07,119 INFO [StoreOpener-953f259f29896493f17802c953dd7fd4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 953f259f29896493f17802c953dd7fd4 2024-11-27T12:24:07,121 INFO [StoreOpener-953f259f29896493f17802c953dd7fd4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 953f259f29896493f17802c953dd7fd4 columnFamilyName C 2024-11-27T12:24:07,121 DEBUG [StoreOpener-953f259f29896493f17802c953dd7fd4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:07,122 INFO [StoreOpener-953f259f29896493f17802c953dd7fd4-1 {}] regionserver.HStore(327): Store=953f259f29896493f17802c953dd7fd4/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T12:24:07,122 DEBUG [RS_OPEN_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 953f259f29896493f17802c953dd7fd4 2024-11-27T12:24:07,123 DEBUG [RS_OPEN_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4 2024-11-27T12:24:07,124 DEBUG [RS_OPEN_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4 2024-11-27T12:24:07,126 DEBUG [RS_OPEN_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 953f259f29896493f17802c953dd7fd4 2024-11-27T12:24:07,126 DEBUG [RS_OPEN_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 953f259f29896493f17802c953dd7fd4 2024-11-27T12:24:07,127 DEBUG [RS_OPEN_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-27T12:24:07,129 DEBUG [RS_OPEN_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 953f259f29896493f17802c953dd7fd4 2024-11-27T12:24:07,132 DEBUG [RS_OPEN_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-27T12:24:07,133 INFO [RS_OPEN_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 953f259f29896493f17802c953dd7fd4; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61932138, jitterRate=-0.0771392285823822}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-27T12:24:07,133 DEBUG [RS_OPEN_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 953f259f29896493f17802c953dd7fd4 2024-11-27T12:24:07,134 DEBUG [RS_OPEN_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 953f259f29896493f17802c953dd7fd4: Running coprocessor pre-open hook at 1732710247107Writing region info on filesystem at 1732710247107Initializing all the Stores at 1732710247110 (+3 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732710247110Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732710247110Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732710247110Cleaning up temporary data from old regions at 1732710247126 (+16 ms)Running coprocessor post-open hooks at 1732710247133 (+7 ms)Region opened successfully at 1732710247134 (+1 ms) 2024-11-27T12:24:07,136 INFO [RS_OPEN_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., pid=6, masterSystemTime=1732710247094 2024-11-27T12:24:07,140 DEBUG [RS_OPEN_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4. 2024-11-27T12:24:07,140 INFO [RS_OPEN_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4. 2024-11-27T12:24:07,141 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=953f259f29896493f17802c953dd7fd4, regionState=OPEN, openSeqNum=2, regionLocation=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:07,145 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 because future has completed 2024-11-27T12:24:07,153 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-27T12:24:07,153 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 in 208 msec 2024-11-27T12:24:07,158 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-27T12:24:07,158 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=953f259f29896493f17802c953dd7fd4, ASSIGN in 383 msec 2024-11-27T12:24:07,160 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-27T12:24:07,160 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732710247160"}]},"ts":"1732710247160"} 2024-11-27T12:24:07,164 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-27T12:24:07,166 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-27T12:24:07,170 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 871 msec 2024-11-27T12:24:07,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-27T12:24:07,461 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-27T12:24:07,469 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@13af8352, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T12:24:07,469 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1ba5af43fd99,36215,-1 for getting cluster id 2024-11-27T12:24:07,469 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T12:24:07,472 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '200978c2-2c41-4f93-b0b5-127ea1024f37' 2024-11-27T12:24:07,473 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T12:24:07,473 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "200978c2-2c41-4f93-b0b5-127ea1024f37" 2024-11-27T12:24:07,474 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@48a04f71, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T12:24:07,474 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1ba5af43fd99,36215,-1] 2024-11-27T12:24:07,475 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T12:24:07,475 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T12:24:07,477 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56418, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T12:24:07,479 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@435bed7b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T12:24:07,479 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T12:24:07,481 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1ba5af43fd99,36251,1732710241631, seqNum=-1] 2024-11-27T12:24:07,481 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T12:24:07,483 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60286, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T12:24:07,485 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-27T12:24:07,487 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is 1ba5af43fd99,36215,1732710240860 2024-11-27T12:24:07,487 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@6bf45604 2024-11-27T12:24:07,489 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-27T12:24:07,491 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56434, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-27T12:24:07,495 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@bd75e6a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T12:24:07,495 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1ba5af43fd99,36215,-1 for getting cluster id 2024-11-27T12:24:07,495 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T12:24:07,497 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '200978c2-2c41-4f93-b0b5-127ea1024f37' 2024-11-27T12:24:07,497 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T12:24:07,497 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "200978c2-2c41-4f93-b0b5-127ea1024f37" 2024-11-27T12:24:07,498 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3b3d2fd6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T12:24:07,498 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1ba5af43fd99,36215,-1] 2024-11-27T12:24:07,498 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T12:24:07,498 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T12:24:07,500 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56452, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T12:24:07,501 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5c860641, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T12:24:07,502 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4d2118b3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T12:24:07,502 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1ba5af43fd99,36215,-1 for getting cluster id 2024-11-27T12:24:07,502 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T12:24:07,504 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '200978c2-2c41-4f93-b0b5-127ea1024f37' 2024-11-27T12:24:07,504 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T12:24:07,504 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "200978c2-2c41-4f93-b0b5-127ea1024f37" 2024-11-27T12:24:07,505 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@787f40de, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T12:24:07,505 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1ba5af43fd99,36215,-1] 2024-11-27T12:24:07,505 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T12:24:07,505 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T12:24:07,507 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56472, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T12:24:07,508 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@740f65e4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T12:24:07,509 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@37b60dcf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T12:24:07,509 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1ba5af43fd99,36215,-1 for getting cluster id 2024-11-27T12:24:07,509 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T12:24:07,512 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '200978c2-2c41-4f93-b0b5-127ea1024f37' 2024-11-27T12:24:07,512 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T12:24:07,513 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "200978c2-2c41-4f93-b0b5-127ea1024f37" 2024-11-27T12:24:07,513 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@69b21050, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T12:24:07,513 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1ba5af43fd99,36215,-1] 2024-11-27T12:24:07,514 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T12:24:07,514 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T12:24:07,515 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56494, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T12:24:07,517 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@79382fcc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T12:24:07,517 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@323acfb6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T12:24:07,518 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1ba5af43fd99,36215,-1 for getting cluster id 2024-11-27T12:24:07,518 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T12:24:07,519 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '200978c2-2c41-4f93-b0b5-127ea1024f37' 2024-11-27T12:24:07,519 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T12:24:07,520 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "200978c2-2c41-4f93-b0b5-127ea1024f37" 2024-11-27T12:24:07,520 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@45101745, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T12:24:07,520 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1ba5af43fd99,36215,-1] 2024-11-27T12:24:07,520 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T12:24:07,521 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T12:24:07,523 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56520, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T12:24:07,524 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5d0d5e57, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T12:24:07,525 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3f3f6ca3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T12:24:07,525 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1ba5af43fd99,36215,-1 for getting cluster id 2024-11-27T12:24:07,525 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T12:24:07,526 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '200978c2-2c41-4f93-b0b5-127ea1024f37' 2024-11-27T12:24:07,526 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T12:24:07,526 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "200978c2-2c41-4f93-b0b5-127ea1024f37" 2024-11-27T12:24:07,527 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3964ad25, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T12:24:07,527 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1ba5af43fd99,36215,-1] 2024-11-27T12:24:07,527 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T12:24:07,527 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T12:24:07,528 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56536, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T12:24:07,529 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@23ea0df3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T12:24:07,531 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@31a68df1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T12:24:07,531 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1ba5af43fd99,36215,-1 for getting cluster id 2024-11-27T12:24:07,531 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T12:24:07,532 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '200978c2-2c41-4f93-b0b5-127ea1024f37' 2024-11-27T12:24:07,532 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T12:24:07,532 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "200978c2-2c41-4f93-b0b5-127ea1024f37" 2024-11-27T12:24:07,533 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@bf4dbe1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T12:24:07,533 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1ba5af43fd99,36215,-1] 2024-11-27T12:24:07,533 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T12:24:07,534 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T12:24:07,535 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56560, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T12:24:07,536 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b6a7b43, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T12:24:07,536 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@667ba1d7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T12:24:07,537 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1ba5af43fd99,36215,-1 for getting cluster id 2024-11-27T12:24:07,537 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T12:24:07,538 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '200978c2-2c41-4f93-b0b5-127ea1024f37' 2024-11-27T12:24:07,538 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T12:24:07,538 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "200978c2-2c41-4f93-b0b5-127ea1024f37" 2024-11-27T12:24:07,539 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6e9550bc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T12:24:07,539 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1ba5af43fd99,36215,-1] 2024-11-27T12:24:07,539 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T12:24:07,539 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T12:24:07,540 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56578, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T12:24:07,541 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2b38edab, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T12:24:07,542 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@20e35624, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T12:24:07,543 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1ba5af43fd99,36215,-1 for getting cluster id 2024-11-27T12:24:07,543 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T12:24:07,544 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '200978c2-2c41-4f93-b0b5-127ea1024f37' 2024-11-27T12:24:07,544 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T12:24:07,544 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "200978c2-2c41-4f93-b0b5-127ea1024f37" 2024-11-27T12:24:07,545 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1020fb6f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T12:24:07,545 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1ba5af43fd99,36215,-1] 2024-11-27T12:24:07,545 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T12:24:07,545 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T12:24:07,547 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56606, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T12:24:07,548 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6c2566c4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T12:24:07,548 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@10c99741, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T12:24:07,549 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1ba5af43fd99,36215,-1 for getting cluster id 2024-11-27T12:24:07,549 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T12:24:07,550 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '200978c2-2c41-4f93-b0b5-127ea1024f37' 2024-11-27T12:24:07,550 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T12:24:07,550 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "200978c2-2c41-4f93-b0b5-127ea1024f37" 2024-11-27T12:24:07,550 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@37a0271, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T12:24:07,550 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1ba5af43fd99,36215,-1] 2024-11-27T12:24:07,551 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T12:24:07,551 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T12:24:07,552 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56620, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T12:24:07,553 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@9402b31, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T12:24:07,558 DEBUG [Thread-154 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T12:24:07,558 DEBUG [Thread-147 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T12:24:07,558 DEBUG [Thread-143 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T12:24:07,558 DEBUG [Thread-158 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T12:24:07,559 DEBUG [Thread-156 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T12:24:07,559 DEBUG [Thread-145 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T12:24:07,559 DEBUG [Thread-151 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T12:24:07,558 DEBUG [Thread-160 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T12:24:07,558 DEBUG [Thread-149 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T12:24:07,561 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1ba5af43fd99,36251,1732710241631, seqNum=-1] 2024-11-27T12:24:07,561 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1ba5af43fd99,36251,1732710241631, seqNum=-1] 2024-11-27T12:24:07,561 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1ba5af43fd99,36251,1732710241631, seqNum=-1] 2024-11-27T12:24:07,561 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1ba5af43fd99,36251,1732710241631, seqNum=-1] 2024-11-27T12:24:07,561 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T12:24:07,561 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T12:24:07,561 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T12:24:07,561 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1ba5af43fd99,36251,1732710241631, seqNum=-1] 2024-11-27T12:24:07,561 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T12:24:07,561 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1ba5af43fd99,36251,1732710241631, seqNum=-1] 2024-11-27T12:24:07,562 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T12:24:07,562 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T12:24:07,563 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1ba5af43fd99,36251,1732710241631, seqNum=-1] 2024-11-27T12:24:07,563 DEBUG [Thread-143 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1ba5af43fd99,36251,1732710241631, seqNum=-1] 2024-11-27T12:24:07,563 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1ba5af43fd99,36251,1732710241631, seqNum=-1] 2024-11-27T12:24:07,563 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T12:24:07,563 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60292, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T12:24:07,563 DEBUG [Thread-143 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T12:24:07,563 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T12:24:07,563 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60298, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T12:24:07,564 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60300, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T12:24:07,564 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60314, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T12:24:07,564 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60316, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T12:24:07,565 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60332, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T12:24:07,565 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60338, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T12:24:07,566 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60348, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T12:24:07,566 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60356, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T12:24:07,571 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2] 2024-11-27T12:24:07,571 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2] 2024-11-27T12:24:07,573 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2] 2024-11-27T12:24:07,574 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2] 2024-11-27T12:24:07,577 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36215 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-27T12:24:07,581 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2] 2024-11-27T12:24:07,583 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2] 2024-11-27T12:24:07,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36215 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees 2024-11-27T12:24:07,585 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2] 2024-11-27T12:24:07,587 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2] 2024-11-27T12:24:07,588 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T12:24:07,589 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2] 2024-11-27T12:24:07,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-27T12:24:07,590 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T12:24:07,592 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T12:24:07,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(8855): Flush requested on 953f259f29896493f17802c953dd7fd4 2024-11-27T12:24:07,670 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T12:24:07,674 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 953f259f29896493f17802c953dd7fd4 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-27T12:24:07,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-27T12:24:07,768 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-27T12:24:07,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4. 2024-11-27T12:24:07,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4. as already flushing 2024-11-27T12:24:07,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4. 2024-11-27T12:24:07,777 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 953f259f29896493f17802c953dd7fd4, NAME => 'TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:24:07,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 953f259f29896493f17802c953dd7fd4, NAME => 'TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:24:07,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36215 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 953f259f29896493f17802c953dd7fd4, NAME => 'TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 953f259f29896493f17802c953dd7fd4, NAME => 'TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:24:07,841 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/.tmp/A/8099951072fb4accbd57bdc559a2bf3d is 50, key is test_row_0/A:col10/1732710247662/Put/seqid=0 2024-11-27T12:24:07,851 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:07,853 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:07,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60300 deadline: 1732710307823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:07,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60298 deadline: 1732710307834, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:07,859 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073741837_1013 (size=12001) 2024-11-27T12:24:07,862 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/.tmp/A/8099951072fb4accbd57bdc559a2bf3d 2024-11-27T12:24:07,862 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:07,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60292 deadline: 1732710307853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:07,864 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:07,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60356 deadline: 1732710307854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:07,868 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:07,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60348 deadline: 1732710307855, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:07,887 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:07,888 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:07,888 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:24:07,889 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:07,889 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:07,889 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:24:07,889 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:07,889 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:07,889 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:24:07,890 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:07,890 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:07,890 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:24:07,890 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:07,890 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:07,890 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:24:07,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-27T12:24:07,969 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/.tmp/B/73374052d1204864af13309c8b4250ed is 50, key is test_row_0/B:col10/1732710247662/Put/seqid=0 2024-11-27T12:24:07,973 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-27T12:24:07,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4. 2024-11-27T12:24:07,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4. as already flushing 2024-11-27T12:24:07,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4. 2024-11-27T12:24:07,983 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 953f259f29896493f17802c953dd7fd4, NAME => 'TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:24:07,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 953f259f29896493f17802c953dd7fd4, NAME => 'TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:24:07,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36215 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 953f259f29896493f17802c953dd7fd4, NAME => 'TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 953f259f29896493f17802c953dd7fd4, NAME => 'TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:24:07,992 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073741838_1014 (size=12001) 2024-11-27T12:24:08,005 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:08,006 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:08,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60348 deadline: 1732710308000, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:08,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60356 deadline: 1732710308000, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:08,007 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:08,007 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:08,007 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:08,007 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:08,007 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:24:08,007 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:24:08,007 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:08,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60292 deadline: 1732710308001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:08,008 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:08,008 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:08,008 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:24:08,008 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:08,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60300 deadline: 1732710308001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:08,009 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:08,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60298 deadline: 1732710307999, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:08,009 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:08,010 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:08,010 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:24:08,010 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:08,010 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:08,010 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:24:08,140 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-27T12:24:08,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4. 2024-11-27T12:24:08,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4. as already flushing 2024-11-27T12:24:08,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4. 2024-11-27T12:24:08,145 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 953f259f29896493f17802c953dd7fd4, NAME => 'TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:24:08,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 953f259f29896493f17802c953dd7fd4, NAME => 'TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:24:08,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36215 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 953f259f29896493f17802c953dd7fd4, NAME => 'TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 953f259f29896493f17802c953dd7fd4, NAME => 'TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:24:08,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-27T12:24:08,223 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:08,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60356 deadline: 1732710308219, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:08,224 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:08,225 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:08,225 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:24:08,225 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:08,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60292 deadline: 1732710308220, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:08,226 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:08,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60300 deadline: 1732710308220, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:08,227 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:08,227 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:08,227 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:24:08,227 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:08,227 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:08,227 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:24:08,228 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:08,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60348 deadline: 1732710308221, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:08,229 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:08,229 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:08,229 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:24:08,230 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:08,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60298 deadline: 1732710308221, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:08,232 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:08,232 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:08,232 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:24:08,301 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-27T12:24:08,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4. 2024-11-27T12:24:08,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4. as already flushing 2024-11-27T12:24:08,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4. 2024-11-27T12:24:08,301 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 953f259f29896493f17802c953dd7fd4, NAME => 'TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:24:08,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 953f259f29896493f17802c953dd7fd4, NAME => 'TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:24:08,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36215 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 953f259f29896493f17802c953dd7fd4, NAME => 'TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 953f259f29896493f17802c953dd7fd4, NAME => 'TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:24:08,396 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/.tmp/B/73374052d1204864af13309c8b4250ed 2024-11-27T12:24:08,444 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/.tmp/C/9994adf73fb04d6b95011b5baf512bfb is 50, key is test_row_0/C:col10/1732710247662/Put/seqid=0 2024-11-27T12:24:08,459 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-27T12:24:08,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4. 2024-11-27T12:24:08,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4. as already flushing 2024-11-27T12:24:08,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4. 2024-11-27T12:24:08,459 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073741839_1015 (size=12001) 2024-11-27T12:24:08,459 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 953f259f29896493f17802c953dd7fd4, NAME => 'TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:24:08,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 953f259f29896493f17802c953dd7fd4, NAME => 'TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:24:08,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36215 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 953f259f29896493f17802c953dd7fd4, NAME => 'TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 953f259f29896493f17802c953dd7fd4, NAME => 'TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:24:08,530 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:08,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60356 deadline: 1732710308529, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:08,532 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:08,532 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:08,532 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:24:08,542 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:08,542 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:08,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60300 deadline: 1732710308539, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:08,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60348 deadline: 1732710308540, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:08,543 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:08,544 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:08,544 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:24:08,544 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:08,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60292 deadline: 1732710308540, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:08,545 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:08,545 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:08,545 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:24:08,545 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:08,545 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:08,546 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:24:08,546 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:08,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60298 deadline: 1732710308540, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:08,547 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:08,547 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:08,547 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:24:08,615 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-27T12:24:08,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4. 2024-11-27T12:24:08,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4. as already flushing 2024-11-27T12:24:08,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4. 2024-11-27T12:24:08,616 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 953f259f29896493f17802c953dd7fd4, NAME => 'TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:24:08,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 953f259f29896493f17802c953dd7fd4, NAME => 'TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:24:08,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36215 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 953f259f29896493f17802c953dd7fd4, NAME => 'TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 953f259f29896493f17802c953dd7fd4, NAME => 'TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:24:08,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-27T12:24:08,773 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-27T12:24:08,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4. 2024-11-27T12:24:08,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4. as already flushing 2024-11-27T12:24:08,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4. 2024-11-27T12:24:08,774 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 953f259f29896493f17802c953dd7fd4, NAME => 'TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:24:08,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 953f259f29896493f17802c953dd7fd4, NAME => 'TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:24:08,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36215 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 953f259f29896493f17802c953dd7fd4, NAME => 'TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 953f259f29896493f17802c953dd7fd4, NAME => 'TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:24:08,862 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/.tmp/C/9994adf73fb04d6b95011b5baf512bfb 2024-11-27T12:24:08,879 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/.tmp/A/8099951072fb4accbd57bdc559a2bf3d as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/A/8099951072fb4accbd57bdc559a2bf3d 2024-11-27T12:24:08,897 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/A/8099951072fb4accbd57bdc559a2bf3d, entries=150, sequenceid=13, filesize=11.7 K 2024-11-27T12:24:08,903 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/.tmp/B/73374052d1204864af13309c8b4250ed as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/B/73374052d1204864af13309c8b4250ed 2024-11-27T12:24:08,916 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/B/73374052d1204864af13309c8b4250ed, entries=150, sequenceid=13, filesize=11.7 K 2024-11-27T12:24:08,919 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/.tmp/C/9994adf73fb04d6b95011b5baf512bfb as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/C/9994adf73fb04d6b95011b5baf512bfb 2024-11-27T12:24:08,929 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-27T12:24:08,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4. 2024-11-27T12:24:08,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4. as already flushing 2024-11-27T12:24:08,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4. 2024-11-27T12:24:08,930 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 953f259f29896493f17802c953dd7fd4, NAME => 'TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:24:08,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 953f259f29896493f17802c953dd7fd4, NAME => 'TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:24:08,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36215 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 953f259f29896493f17802c953dd7fd4, NAME => 'TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 953f259f29896493f17802c953dd7fd4, NAME => 'TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:24:08,936 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/C/9994adf73fb04d6b95011b5baf512bfb, entries=150, sequenceid=13, filesize=11.7 K 2024-11-27T12:24:08,944 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=154.31 KB/158010 for 953f259f29896493f17802c953dd7fd4 in 1268ms, sequenceid=13, compaction requested=false 2024-11-27T12:24:08,945 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-27T12:24:08,947 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 953f259f29896493f17802c953dd7fd4: 2024-11-27T12:24:09,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(8855): Flush requested on 953f259f29896493f17802c953dd7fd4 2024-11-27T12:24:09,043 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T12:24:09,043 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 953f259f29896493f17802c953dd7fd4 3/3 column families, dataSize=161.02 KB heapSize=422.63 KB 2024-11-27T12:24:09,064 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/.tmp/A/64eaaaf2596d48f48d94dde59111acdc is 50, key is test_row_0/A:col10/1732710249039/Put/seqid=0 2024-11-27T12:24:09,077 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:09,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60300 deadline: 1732710309061, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:09,079 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:09,079 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:09,079 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:24:09,079 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:09,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60292 deadline: 1732710309068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:09,080 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:09,080 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:09,080 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:24:09,084 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:09,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60348 deadline: 1732710309074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:09,086 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:09,086 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:09,086 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:24:09,087 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-27T12:24:09,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4. 2024-11-27T12:24:09,087 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:09,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4. as already flushing 2024-11-27T12:24:09,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60298 deadline: 1732710309080, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:09,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4. 2024-11-27T12:24:09,088 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:09,088 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:09,088 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:24:09,089 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:09,089 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 953f259f29896493f17802c953dd7fd4, NAME => 'TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:24:09,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60356 deadline: 1732710309080, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:09,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 953f259f29896493f17802c953dd7fd4, NAME => 'TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:24:09,090 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:09,090 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:09,090 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:24:09,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36215 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 953f259f29896493f17802c953dd7fd4, NAME => 'TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 953f259f29896493f17802c953dd7fd4, NAME => 'TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:24:09,095 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073741840_1016 (size=14341) 2024-11-27T12:24:09,098 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/.tmp/A/64eaaaf2596d48f48d94dde59111acdc 2024-11-27T12:24:09,122 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/.tmp/B/a6c5abf3e4774096bf5225521e4820dc is 50, key is test_row_0/B:col10/1732710249039/Put/seqid=0 2024-11-27T12:24:09,134 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073741841_1017 (size=12001) 2024-11-27T12:24:09,136 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/.tmp/B/a6c5abf3e4774096bf5225521e4820dc 2024-11-27T12:24:09,156 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/.tmp/C/1d99c02af3404b978d18793ad3e3a6dc is 50, key is test_row_0/C:col10/1732710249039/Put/seqid=0 2024-11-27T12:24:09,171 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073741842_1018 (size=12001) 2024-11-27T12:24:09,172 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/.tmp/C/1d99c02af3404b978d18793ad3e3a6dc 2024-11-27T12:24:09,187 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/.tmp/A/64eaaaf2596d48f48d94dde59111acdc as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/A/64eaaaf2596d48f48d94dde59111acdc 2024-11-27T12:24:09,200 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:09,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60292 deadline: 1732710309189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:09,200 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:09,201 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:09,201 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:09,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60348 deadline: 1732710309190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:09,201 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:24:09,201 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:09,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60300 deadline: 1732710309189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:09,202 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:09,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60298 deadline: 1732710309199, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:09,202 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:09,202 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:09,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60356 deadline: 1732710309200, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:09,202 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:09,203 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:24:09,203 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:09,203 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:09,203 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:24:09,203 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:09,203 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:09,203 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:09,203 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:09,203 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:24:09,203 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:24:09,210 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/A/64eaaaf2596d48f48d94dde59111acdc, entries=200, sequenceid=40, filesize=14.0 K 2024-11-27T12:24:09,213 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/.tmp/B/a6c5abf3e4774096bf5225521e4820dc as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/B/a6c5abf3e4774096bf5225521e4820dc 2024-11-27T12:24:09,228 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/B/a6c5abf3e4774096bf5225521e4820dc, entries=150, sequenceid=40, filesize=11.7 K 2024-11-27T12:24:09,232 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/.tmp/C/1d99c02af3404b978d18793ad3e3a6dc as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/C/1d99c02af3404b978d18793ad3e3a6dc 2024-11-27T12:24:09,244 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-27T12:24:09,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4. 2024-11-27T12:24:09,244 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/C/1d99c02af3404b978d18793ad3e3a6dc, entries=150, sequenceid=40, filesize=11.7 K 2024-11-27T12:24:09,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4. as already flushing 2024-11-27T12:24:09,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4. 2024-11-27T12:24:09,245 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 953f259f29896493f17802c953dd7fd4, NAME => 'TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:24:09,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 953f259f29896493f17802c953dd7fd4, NAME => 'TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:24:09,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36215 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 953f259f29896493f17802c953dd7fd4, NAME => 'TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 953f259f29896493f17802c953dd7fd4, NAME => 'TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:24:09,247 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=46.96 KB/48090 for 953f259f29896493f17802c953dd7fd4 in 204ms, sequenceid=40, compaction requested=false 2024-11-27T12:24:09,247 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 953f259f29896493f17802c953dd7fd4: 2024-11-27T12:24:09,399 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-27T12:24:09,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4. 2024-11-27T12:24:09,400 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2902): Flushing 953f259f29896493f17802c953dd7fd4 3/3 column families, dataSize=46.96 KB heapSize=123.80 KB 2024-11-27T12:24:09,413 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4. as already flushing 2024-11-27T12:24:09,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(8855): Flush requested on 953f259f29896493f17802c953dd7fd4 2024-11-27T12:24:09,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/.tmp/A/e3c3fedbfa7147cfb5a23ccf66090521 is 50, key is test_row_0/A:col10/1732710249069/Put/seqid=0 2024-11-27T12:24:09,428 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073741843_1019 (size=12001) 2024-11-27T12:24:09,484 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:09,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60298 deadline: 1732710309476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:09,485 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:09,485 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:09,485 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:09,485 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:24:09,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60292 deadline: 1732710309476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:09,486 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:09,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60348 deadline: 1732710309476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:09,487 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:09,487 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:09,487 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:24:09,487 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:09,487 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:09,487 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:24:09,487 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:09,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60356 deadline: 1732710309480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:09,489 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:09,489 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:09,489 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:09,489 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:24:09,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60300 deadline: 1732710309480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:09,489 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:09,490 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:09,490 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:24:09,591 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:09,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60298 deadline: 1732710309588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:09,592 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:09,592 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:09,592 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:24:09,604 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:09,604 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:09,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60348 deadline: 1732710309598, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:09,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60292 deadline: 1732710309598, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:09,605 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:09,605 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:09,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60300 deadline: 1732710309599, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:09,606 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:09,606 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:09,606 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:09,606 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:09,606 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:24:09,606 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:24:09,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60356 deadline: 1732710309599, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:09,607 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:09,607 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:09,607 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:09,607 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:09,607 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:24:09,607 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:24:09,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-27T12:24:09,800 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:09,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60298 deadline: 1732710309799, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:09,801 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:09,801 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:09,801 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:24:09,811 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:09,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60348 deadline: 1732710309808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:09,812 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:09,812 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:09,812 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:24:09,823 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:09,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60300 deadline: 1732710309819, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:09,825 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:09,825 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:09,825 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:24:09,825 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:09,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60292 deadline: 1732710309819, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:09,827 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:09,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60356 deadline: 1732710309821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:09,828 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:09,828 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:09,828 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:24:09,828 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:09,828 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:09,829 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:24:09,830 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/.tmp/A/e3c3fedbfa7147cfb5a23ccf66090521 2024-11-27T12:24:09,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/.tmp/B/0e7d5f462461482e970041c4a1fb2dac is 50, key is test_row_0/B:col10/1732710249069/Put/seqid=0 2024-11-27T12:24:09,879 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073741844_1020 (size=12001) 2024-11-27T12:24:10,110 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:10,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60298 deadline: 1732710310108, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:10,111 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:10,111 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:10,111 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:24:10,121 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:10,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60348 deadline: 1732710310118, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:10,122 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:10,122 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:10,122 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:24:10,131 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:10,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60300 deadline: 1732710310128, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:10,133 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:10,133 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:10,133 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:24:10,139 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:10,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60356 deadline: 1732710310139, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:10,140 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:10,140 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:10,141 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:10,141 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:24:10,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60292 deadline: 1732710310139, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:10,144 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:10,145 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:10,145 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:24:10,281 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/.tmp/B/0e7d5f462461482e970041c4a1fb2dac 2024-11-27T12:24:10,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/.tmp/C/ca75ebc244404644b7ddbdb9b875f012 is 50, key is test_row_0/C:col10/1732710249069/Put/seqid=0 2024-11-27T12:24:10,331 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073741845_1021 (size=12001) 2024-11-27T12:24:10,337 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/.tmp/C/ca75ebc244404644b7ddbdb9b875f012 2024-11-27T12:24:10,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/.tmp/A/e3c3fedbfa7147cfb5a23ccf66090521 as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/A/e3c3fedbfa7147cfb5a23ccf66090521 2024-11-27T12:24:10,375 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/A/e3c3fedbfa7147cfb5a23ccf66090521, entries=150, sequenceid=50, filesize=11.7 K 2024-11-27T12:24:10,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/.tmp/B/0e7d5f462461482e970041c4a1fb2dac as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/B/0e7d5f462461482e970041c4a1fb2dac 2024-11-27T12:24:10,404 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/B/0e7d5f462461482e970041c4a1fb2dac, entries=150, sequenceid=50, filesize=11.7 K 2024-11-27T12:24:10,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/.tmp/C/ca75ebc244404644b7ddbdb9b875f012 as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/C/ca75ebc244404644b7ddbdb9b875f012 2024-11-27T12:24:10,425 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/C/ca75ebc244404644b7ddbdb9b875f012, entries=150, sequenceid=50, filesize=11.7 K 2024-11-27T12:24:10,429 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~123.75 KB/126720, currentSize=161.02 KB/164880 for 953f259f29896493f17802c953dd7fd4 in 1029ms, sequenceid=50, compaction requested=true 2024-11-27T12:24:10,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2603): Flush status journal for 953f259f29896493f17802c953dd7fd4: 2024-11-27T12:24:10,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4. 2024-11-27T12:24:10,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=8 2024-11-27T12:24:10,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36215 {}] master.HMaster(4169): Remote procedure done, pid=8 2024-11-27T12:24:10,437 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-27T12:24:10,437 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.8410 sec 2024-11-27T12:24:10,441 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees in 2.8600 sec 2024-11-27T12:24:10,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(8855): Flush requested on 953f259f29896493f17802c953dd7fd4 2024-11-27T12:24:10,623 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T12:24:10,623 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 953f259f29896493f17802c953dd7fd4 3/3 column families, dataSize=167.72 KB heapSize=440.20 KB 2024-11-27T12:24:10,625 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-27T12:24:10,652 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/.tmp/A/a6c580332c9042ce80de73f6a68a90f5 is 50, key is test_row_0/A:col10/1732710249433/Put/seqid=0 2024-11-27T12:24:10,655 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:10,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60292 deadline: 1732710310648, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:10,656 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:10,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60356 deadline: 1732710310648, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:10,656 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:10,656 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:10,656 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:24:10,656 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:10,657 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:10,657 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:24:10,657 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:10,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60300 deadline: 1732710310650, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:10,658 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:10,658 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:10,658 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:10,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60298 deadline: 1732710310653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:10,658 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:24:10,659 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:10,659 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:10,659 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:24:10,661 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:10,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60348 deadline: 1732710310657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:10,663 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:10,663 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:10,663 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:24:10,671 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073741846_1022 (size=12001) 2024-11-27T12:24:10,675 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=78 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/.tmp/A/a6c580332c9042ce80de73f6a68a90f5 2024-11-27T12:24:10,698 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/.tmp/B/c146c96db80c4b2985d7601d28386162 is 50, key is test_row_0/B:col10/1732710249433/Put/seqid=0 2024-11-27T12:24:10,730 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073741847_1023 (size=12001) 2024-11-27T12:24:10,771 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:10,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60300 deadline: 1732710310768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:10,772 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:10,772 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:10,772 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:10,773 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:24:10,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60348 deadline: 1732710310769, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:10,773 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:10,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60298 deadline: 1732710310769, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:10,774 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:10,774 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:10,774 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:24:10,776 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:10,776 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:10,776 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:24:10,981 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:10,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60298 deadline: 1732710310978, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:10,982 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:10,982 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:10,982 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:10,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60300 deadline: 1732710310978, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:10,982 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:24:10,984 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:10,984 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:10,984 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:24:10,993 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:10,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60348 deadline: 1732710310988, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:10,994 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:10,995 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:10,995 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:24:11,133 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=78 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/.tmp/B/c146c96db80c4b2985d7601d28386162 2024-11-27T12:24:11,171 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/.tmp/C/551ca4808fab40b493220a40b15312bd is 50, key is test_row_0/C:col10/1732710249433/Put/seqid=0 2024-11-27T12:24:11,204 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073741848_1024 (size=12001) 2024-11-27T12:24:11,206 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=78 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/.tmp/C/551ca4808fab40b493220a40b15312bd 2024-11-27T12:24:11,219 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/.tmp/A/a6c580332c9042ce80de73f6a68a90f5 as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/A/a6c580332c9042ce80de73f6a68a90f5 2024-11-27T12:24:11,238 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/A/a6c580332c9042ce80de73f6a68a90f5, entries=150, sequenceid=78, filesize=11.7 K 2024-11-27T12:24:11,241 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/.tmp/B/c146c96db80c4b2985d7601d28386162 as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/B/c146c96db80c4b2985d7601d28386162 2024-11-27T12:24:11,259 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/B/c146c96db80c4b2985d7601d28386162, entries=150, sequenceid=78, filesize=11.7 K 2024-11-27T12:24:11,261 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/.tmp/C/551ca4808fab40b493220a40b15312bd as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/C/551ca4808fab40b493220a40b15312bd 2024-11-27T12:24:11,279 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/C/551ca4808fab40b493220a40b15312bd, entries=150, sequenceid=78, filesize=11.7 K 2024-11-27T12:24:11,281 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~167.72 KB/171750, heapSize ~440.16 KB/450720, currentSize=33.54 KB/34350 for 953f259f29896493f17802c953dd7fd4 in 658ms, sequenceid=78, compaction requested=true 2024-11-27T12:24:11,281 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 953f259f29896493f17802c953dd7fd4: 2024-11-27T12:24:11,286 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 953f259f29896493f17802c953dd7fd4:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T12:24:11,288 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T12:24:11,288 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-27T12:24:11,289 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-27T12:24:11,289 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 953f259f29896493f17802c953dd7fd4:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T12:24:11,289 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T12:24:11,289 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 953f259f29896493f17802c953dd7fd4:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T12:24:11,289 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T12:24:11,293 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-27T12:24:11,294 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HStore(1541): 953f259f29896493f17802c953dd7fd4/B is initiating minor compaction (all files) 2024-11-27T12:24:11,295 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 953f259f29896493f17802c953dd7fd4/B in TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4. 2024-11-27T12:24:11,295 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/B/73374052d1204864af13309c8b4250ed, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/B/a6c5abf3e4774096bf5225521e4820dc, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/B/0e7d5f462461482e970041c4a1fb2dac, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/B/c146c96db80c4b2985d7601d28386162] into tmpdir=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/.tmp, totalSize=46.9 K 2024-11-27T12:24:11,298 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.Compactor(225): Compacting 73374052d1204864af13309c8b4250ed, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1732710247649 2024-11-27T12:24:11,300 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50344 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-27T12:24:11,300 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HStore(1541): 953f259f29896493f17802c953dd7fd4/A is initiating minor compaction (all files) 2024-11-27T12:24:11,300 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 953f259f29896493f17802c953dd7fd4/A in TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4. 2024-11-27T12:24:11,300 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/A/8099951072fb4accbd57bdc559a2bf3d, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/A/64eaaaf2596d48f48d94dde59111acdc, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/A/e3c3fedbfa7147cfb5a23ccf66090521, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/A/a6c580332c9042ce80de73f6a68a90f5] into tmpdir=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/.tmp, totalSize=49.2 K 2024-11-27T12:24:11,302 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.Compactor(225): Compacting a6c5abf3e4774096bf5225521e4820dc, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1732710247812 2024-11-27T12:24:11,303 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8099951072fb4accbd57bdc559a2bf3d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1732710247649 2024-11-27T12:24:11,303 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.Compactor(225): Compacting 0e7d5f462461482e970041c4a1fb2dac, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1732710249069 2024-11-27T12:24:11,304 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.Compactor(225): Compacting c146c96db80c4b2985d7601d28386162, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1732710249433 2024-11-27T12:24:11,304 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] compactions.Compactor(225): Compacting 64eaaaf2596d48f48d94dde59111acdc, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1732710247812 2024-11-27T12:24:11,306 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] compactions.Compactor(225): Compacting e3c3fedbfa7147cfb5a23ccf66090521, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1732710249069 2024-11-27T12:24:11,309 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] compactions.Compactor(225): Compacting a6c580332c9042ce80de73f6a68a90f5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1732710249433 2024-11-27T12:24:11,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(8855): Flush requested on 953f259f29896493f17802c953dd7fd4 2024-11-27T12:24:11,322 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T12:24:11,323 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 953f259f29896493f17802c953dd7fd4 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-27T12:24:11,339 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/.tmp/A/68e4df30616f480c975bf82bc491a7bc is 50, key is test_row_0/A:col10/1732710250651/Put/seqid=0 2024-11-27T12:24:11,359 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 953f259f29896493f17802c953dd7fd4#A#compaction#13 average throughput is 0.50 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-27T12:24:11,361 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/.tmp/A/c77e507f29524cd1b12ee7dfed66f019 is 50, key is test_row_0/A:col10/1732710249433/Put/seqid=0 2024-11-27T12:24:11,366 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 953f259f29896493f17802c953dd7fd4#B#compaction#14 average throughput is 0.41 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T12:24:11,367 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/.tmp/B/a0f5d8167eeb4429b13790a4e760f1bd is 50, key is test_row_0/B:col10/1732710249433/Put/seqid=0 2024-11-27T12:24:11,382 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-27T12:24:11,392 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073741849_1025 (size=21365) 2024-11-27T12:24:11,395 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073741850_1026 (size=12139) 2024-11-27T12:24:11,410 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073741851_1027 (size=12139) 2024-11-27T12:24:11,423 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/.tmp/B/a0f5d8167eeb4429b13790a4e760f1bd as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/B/a0f5d8167eeb4429b13790a4e760f1bd 2024-11-27T12:24:11,447 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 953f259f29896493f17802c953dd7fd4/B of 953f259f29896493f17802c953dd7fd4 into a0f5d8167eeb4429b13790a4e760f1bd(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T12:24:11,447 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 953f259f29896493f17802c953dd7fd4: 2024-11-27T12:24:11,450 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., storeName=953f259f29896493f17802c953dd7fd4/B, priority=12, startTime=1732710251288; duration=0sec 2024-11-27T12:24:11,451 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T12:24:11,451 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 953f259f29896493f17802c953dd7fd4:B 2024-11-27T12:24:11,451 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-27T12:24:11,455 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-27T12:24:11,456 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HStore(1541): 953f259f29896493f17802c953dd7fd4/C is initiating minor compaction (all files) 2024-11-27T12:24:11,456 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 953f259f29896493f17802c953dd7fd4/C in TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4. 2024-11-27T12:24:11,456 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/C/9994adf73fb04d6b95011b5baf512bfb, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/C/1d99c02af3404b978d18793ad3e3a6dc, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/C/ca75ebc244404644b7ddbdb9b875f012, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/C/551ca4808fab40b493220a40b15312bd] into tmpdir=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/.tmp, totalSize=46.9 K 2024-11-27T12:24:11,457 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.Compactor(225): Compacting 9994adf73fb04d6b95011b5baf512bfb, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1732710247649 2024-11-27T12:24:11,458 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.Compactor(225): Compacting 1d99c02af3404b978d18793ad3e3a6dc, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1732710247812 2024-11-27T12:24:11,459 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.Compactor(225): Compacting ca75ebc244404644b7ddbdb9b875f012, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1732710249069 2024-11-27T12:24:11,460 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.Compactor(225): Compacting 551ca4808fab40b493220a40b15312bd, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1732710249433 2024-11-27T12:24:11,467 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-27T12:24:11,468 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-27T12:24:11,470 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-27T12:24:11,470 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-27T12:24:11,471 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-27T12:24:11,471 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-27T12:24:11,472 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-27T12:24:11,472 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-27T12:24:11,491 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 953f259f29896493f17802c953dd7fd4#C#compaction#15 average throughput is 0.82 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T12:24:11,492 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/.tmp/C/e439ae32af0b4328bf72d5106a7e9cbe is 50, key is test_row_0/C:col10/1732710249433/Put/seqid=0 2024-11-27T12:24:11,527 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073741852_1028 (size=12139) 2024-11-27T12:24:11,531 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:11,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60348 deadline: 1732710311515, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:11,532 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:11,532 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:11,533 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:11,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60300 deadline: 1732710311518, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:11,533 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:24:11,533 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:11,533 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:11,533 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:24:11,534 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:11,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60298 deadline: 1732710311521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:11,535 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:11,535 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:11,535 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:24:11,643 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:11,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60300 deadline: 1732710311638, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:11,644 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:11,644 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:11,644 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:24:11,645 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:11,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60298 deadline: 1732710311638, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:11,646 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:11,646 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:11,646 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:24:11,647 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:11,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60348 deadline: 1732710311638, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:11,648 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:11,648 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:11,648 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:24:11,671 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:11,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60292 deadline: 1732710311668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:11,672 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:11,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60356 deadline: 1732710311668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:11,675 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 1ba5af43fd99,36251,1732710241631 for 'test_row_0' in 953f259f29896493f17802c953dd7fd4 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2193 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T12:24:11,675 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 1ba5af43fd99,36251,1732710241631 for 'test_row_2' in 953f259f29896493f17802c953dd7fd4 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2197 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T12:24:11,677 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:11,677 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:11,677 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:11,677 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:11,677 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:24:11,677 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:24:11,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-27T12:24:11,759 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T12:24:11,760 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36215 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-27T12:24:11,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36215 {}] procedure2.ProcedureExecutor(1139): Stored pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees 2024-11-27T12:24:11,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-27T12:24:11,764 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T12:24:11,766 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T12:24:11,766 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T12:24:11,792 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/.tmp/A/68e4df30616f480c975bf82bc491a7bc 2024-11-27T12:24:11,819 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/.tmp/A/c77e507f29524cd1b12ee7dfed66f019 as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/A/c77e507f29524cd1b12ee7dfed66f019 2024-11-27T12:24:11,830 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/.tmp/B/048712a3d3244a8799d60dc5fd1312e8 is 50, key is test_row_0/B:col10/1732710250651/Put/seqid=0 2024-11-27T12:24:11,834 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 953f259f29896493f17802c953dd7fd4/A of 953f259f29896493f17802c953dd7fd4 into c77e507f29524cd1b12ee7dfed66f019(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T12:24:11,834 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 953f259f29896493f17802c953dd7fd4: 2024-11-27T12:24:11,834 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., storeName=953f259f29896493f17802c953dd7fd4/A, priority=12, startTime=1732710251283; duration=0sec 2024-11-27T12:24:11,835 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T12:24:11,835 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 953f259f29896493f17802c953dd7fd4:A 2024-11-27T12:24:11,853 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:11,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60300 deadline: 1732710311849, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:11,855 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:11,855 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:11,855 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:24:11,858 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073741853_1029 (size=12001) 2024-11-27T12:24:11,860 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:11,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60298 deadline: 1732710311858, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:11,861 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:11,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60348 deadline: 1732710311858, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:11,862 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:11,863 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:11,863 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:11,863 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:24:11,863 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:11,863 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:24:11,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-27T12:24:11,921 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-27T12:24:11,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4. 2024-11-27T12:24:11,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4. as already flushing 2024-11-27T12:24:11,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4. 2024-11-27T12:24:11,922 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 953f259f29896493f17802c953dd7fd4, NAME => 'TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:24:11,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 953f259f29896493f17802c953dd7fd4, NAME => 'TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:24:11,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36215 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 953f259f29896493f17802c953dd7fd4, NAME => 'TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 953f259f29896493f17802c953dd7fd4, NAME => 'TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:24:11,942 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/.tmp/C/e439ae32af0b4328bf72d5106a7e9cbe as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/C/e439ae32af0b4328bf72d5106a7e9cbe 2024-11-27T12:24:11,956 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 953f259f29896493f17802c953dd7fd4/C of 953f259f29896493f17802c953dd7fd4 into e439ae32af0b4328bf72d5106a7e9cbe(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T12:24:11,957 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 953f259f29896493f17802c953dd7fd4: 2024-11-27T12:24:11,957 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., storeName=953f259f29896493f17802c953dd7fd4/C, priority=12, startTime=1732710251289; duration=0sec 2024-11-27T12:24:11,957 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T12:24:11,957 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 953f259f29896493f17802c953dd7fd4:C 2024-11-27T12:24:12,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-27T12:24:12,078 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-27T12:24:12,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4. 2024-11-27T12:24:12,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4. as already flushing 2024-11-27T12:24:12,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4. 2024-11-27T12:24:12,079 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 953f259f29896493f17802c953dd7fd4, NAME => 'TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:24:12,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 953f259f29896493f17802c953dd7fd4, NAME => 'TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:24:12,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36215 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 953f259f29896493f17802c953dd7fd4, NAME => 'TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 953f259f29896493f17802c953dd7fd4, NAME => 'TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:24:12,159 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:12,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60300 deadline: 1732710312158, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:12,160 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:12,160 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:12,160 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:24:12,169 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:12,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60298 deadline: 1732710312168, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:12,171 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:12,171 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:12,171 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:24:12,171 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:12,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60348 deadline: 1732710312168, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:12,173 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:12,173 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:12,173 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:24:12,235 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-27T12:24:12,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4. 2024-11-27T12:24:12,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4. as already flushing 2024-11-27T12:24:12,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4. 2024-11-27T12:24:12,236 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 953f259f29896493f17802c953dd7fd4, NAME => 'TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:24:12,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 953f259f29896493f17802c953dd7fd4, NAME => 'TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:24:12,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36215 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 953f259f29896493f17802c953dd7fd4, NAME => 'TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 953f259f29896493f17802c953dd7fd4, NAME => 'TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:24:12,259 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/.tmp/B/048712a3d3244a8799d60dc5fd1312e8 2024-11-27T12:24:12,292 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/.tmp/C/e0864f52a7ca4630b2b957d17bded558 is 50, key is test_row_0/C:col10/1732710250651/Put/seqid=0 2024-11-27T12:24:12,343 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073741854_1030 (size=12001) 2024-11-27T12:24:12,345 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/.tmp/C/e0864f52a7ca4630b2b957d17bded558 2024-11-27T12:24:12,362 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/.tmp/A/68e4df30616f480c975bf82bc491a7bc as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/A/68e4df30616f480c975bf82bc491a7bc 2024-11-27T12:24:12,376 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/A/68e4df30616f480c975bf82bc491a7bc, entries=350, sequenceid=90, filesize=20.9 K 2024-11-27T12:24:12,380 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/.tmp/B/048712a3d3244a8799d60dc5fd1312e8 as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/B/048712a3d3244a8799d60dc5fd1312e8 2024-11-27T12:24:12,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-27T12:24:12,392 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-27T12:24:12,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4. 2024-11-27T12:24:12,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4. as already flushing 2024-11-27T12:24:12,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4. 2024-11-27T12:24:12,393 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 953f259f29896493f17802c953dd7fd4, NAME => 'TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:24:12,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 953f259f29896493f17802c953dd7fd4, NAME => 'TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:24:12,394 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/B/048712a3d3244a8799d60dc5fd1312e8, entries=150, sequenceid=90, filesize=11.7 K 2024-11-27T12:24:12,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36215 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 953f259f29896493f17802c953dd7fd4, NAME => 'TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 953f259f29896493f17802c953dd7fd4, NAME => 'TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:24:12,397 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/.tmp/C/e0864f52a7ca4630b2b957d17bded558 as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/C/e0864f52a7ca4630b2b957d17bded558 2024-11-27T12:24:12,418 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/C/e0864f52a7ca4630b2b957d17bded558, entries=150, sequenceid=90, filesize=11.7 K 2024-11-27T12:24:12,425 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 953f259f29896493f17802c953dd7fd4 in 1102ms, sequenceid=90, compaction requested=false 2024-11-27T12:24:12,425 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 953f259f29896493f17802c953dd7fd4: 2024-11-27T12:24:12,549 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-27T12:24:12,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4. 2024-11-27T12:24:12,550 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2902): Flushing 953f259f29896493f17802c953dd7fd4 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-27T12:24:12,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/.tmp/A/e56e87dcf7ce477f9248e78c39bfd4ac is 50, key is test_row_0/A:col10/1732710251517/Put/seqid=0 2024-11-27T12:24:12,601 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073741855_1031 (size=12001) 2024-11-27T12:24:12,605 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=117 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/.tmp/A/e56e87dcf7ce477f9248e78c39bfd4ac 2024-11-27T12:24:12,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/.tmp/B/2a35b97e787c4035a00a16b41757adc4 is 50, key is test_row_0/B:col10/1732710251517/Put/seqid=0 2024-11-27T12:24:12,658 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073741856_1032 (size=12001) 2024-11-27T12:24:12,659 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=117 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/.tmp/B/2a35b97e787c4035a00a16b41757adc4 2024-11-27T12:24:12,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(8855): Flush requested on 953f259f29896493f17802c953dd7fd4 2024-11-27T12:24:12,674 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4. as already flushing 2024-11-27T12:24:12,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/.tmp/C/d047d654ce40415780ab3e93779afec5 is 50, key is test_row_0/C:col10/1732710251517/Put/seqid=0 2024-11-27T12:24:12,734 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073741857_1033 (size=12001) 2024-11-27T12:24:12,738 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:12,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60298 deadline: 1732710312732, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:12,740 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:12,740 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:12,740 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:24:12,742 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:12,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60300 deadline: 1732710312733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:12,745 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:12,745 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:12,745 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:24:12,749 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:12,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60348 deadline: 1732710312747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:12,750 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:12,751 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:12,751 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:24:12,856 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:12,857 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:12,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60298 deadline: 1732710312849, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:12,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60300 deadline: 1732710312849, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:12,861 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:12,861 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:12,861 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:24:12,861 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:12,861 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:12,861 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:24:12,868 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:12,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60348 deadline: 1732710312860, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:12,869 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:12,870 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:12,870 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:24:12,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-27T12:24:13,074 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:13,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60298 deadline: 1732710313069, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:13,074 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:13,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60300 deadline: 1732710313069, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:13,077 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:13,077 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:13,077 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:24:13,077 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:13,078 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:13,078 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:24:13,080 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:13,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60348 deadline: 1732710313078, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:13,083 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:13,083 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:13,083 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:24:13,135 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=117 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/.tmp/C/d047d654ce40415780ab3e93779afec5 2024-11-27T12:24:13,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/.tmp/A/e56e87dcf7ce477f9248e78c39bfd4ac as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/A/e56e87dcf7ce477f9248e78c39bfd4ac 2024-11-27T12:24:13,161 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/A/e56e87dcf7ce477f9248e78c39bfd4ac, entries=150, sequenceid=117, filesize=11.7 K 2024-11-27T12:24:13,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/.tmp/B/2a35b97e787c4035a00a16b41757adc4 as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/B/2a35b97e787c4035a00a16b41757adc4 2024-11-27T12:24:13,180 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/B/2a35b97e787c4035a00a16b41757adc4, entries=150, sequenceid=117, filesize=11.7 K 2024-11-27T12:24:13,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/.tmp/C/d047d654ce40415780ab3e93779afec5 as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/C/d047d654ce40415780ab3e93779afec5 2024-11-27T12:24:13,206 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/C/d047d654ce40415780ab3e93779afec5, entries=150, sequenceid=117, filesize=11.7 K 2024-11-27T12:24:13,210 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=67.09 KB/68700 for 953f259f29896493f17802c953dd7fd4 in 661ms, sequenceid=117, compaction requested=true 2024-11-27T12:24:13,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2603): Flush status journal for 953f259f29896493f17802c953dd7fd4: 2024-11-27T12:24:13,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4. 2024-11-27T12:24:13,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=10 2024-11-27T12:24:13,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36215 {}] master.HMaster(4169): Remote procedure done, pid=10 2024-11-27T12:24:13,218 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=9 2024-11-27T12:24:13,219 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4480 sec 2024-11-27T12:24:13,223 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees in 1.4600 sec 2024-11-27T12:24:13,405 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T12:24:13,405 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 953f259f29896493f17802c953dd7fd4 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-27T12:24:13,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(8855): Flush requested on 953f259f29896493f17802c953dd7fd4 2024-11-27T12:24:13,420 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/.tmp/A/6a05cb9776084131a562fd3e67e8f3cb is 50, key is test_row_0/A:col10/1732710253395/Put/seqid=0 2024-11-27T12:24:13,486 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073741858_1034 (size=16881) 2024-11-27T12:24:13,502 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:13,502 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:13,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60348 deadline: 1732710313493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:13,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60298 deadline: 1732710313496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:13,503 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:13,503 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:13,503 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:24:13,503 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:13,504 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:13,504 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:24:13,504 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:13,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60300 deadline: 1732710313499, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:13,505 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:13,505 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:13,505 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:24:13,616 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:13,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60348 deadline: 1732710313609, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:13,617 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:13,617 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:13,617 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:24:13,618 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:13,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60298 deadline: 1732710313610, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:13,619 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:13,619 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:13,619 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:24:13,617 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:13,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60300 deadline: 1732710313609, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:13,630 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:13,630 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:13,630 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:24:13,702 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:13,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60292 deadline: 1732710313698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:13,703 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:13,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60356 deadline: 1732710313698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:13,703 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 1ba5af43fd99,36251,1732710241631 for 'test_row_2' in 953f259f29896493f17802c953dd7fd4 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4228 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T12:24:13,704 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:13,704 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:13,704 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:24:13,704 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 1ba5af43fd99,36251,1732710241631 for 'test_row_0' in 953f259f29896493f17802c953dd7fd4 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4225 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T12:24:13,705 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:13,705 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:13,705 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:24:13,828 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:13,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60348 deadline: 1732710313828, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:13,832 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:13,832 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:13,832 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:24:13,833 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:13,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60298 deadline: 1732710313828, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:13,836 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:13,836 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:13,836 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:24:13,839 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:13,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60300 deadline: 1732710313839, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:13,845 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:13,845 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:13,845 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:24:13,888 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=133 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/.tmp/A/6a05cb9776084131a562fd3e67e8f3cb 2024-11-27T12:24:13,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-27T12:24:13,919 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T12:24:13,920 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36215 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-27T12:24:13,924 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/.tmp/B/62133eb7cc03471aa362cef0dbd6e620 is 50, key is test_row_0/B:col10/1732710253395/Put/seqid=0 2024-11-27T12:24:13,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36215 {}] procedure2.ProcedureExecutor(1139): Stored pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees 2024-11-27T12:24:13,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-27T12:24:13,934 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T12:24:13,936 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T12:24:13,936 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T12:24:13,958 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073741859_1035 (size=12151) 2024-11-27T12:24:13,961 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=133 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/.tmp/B/62133eb7cc03471aa362cef0dbd6e620 2024-11-27T12:24:13,982 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/.tmp/C/20ba43d95b3449329b7c56e9754ba8c9 is 50, key is test_row_0/C:col10/1732710253395/Put/seqid=0 2024-11-27T12:24:14,019 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073741860_1036 (size=12151) 2024-11-27T12:24:14,021 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=133 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/.tmp/C/20ba43d95b3449329b7c56e9754ba8c9 2024-11-27T12:24:14,037 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/.tmp/A/6a05cb9776084131a562fd3e67e8f3cb as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/A/6a05cb9776084131a562fd3e67e8f3cb 2024-11-27T12:24:14,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-27T12:24:14,062 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/A/6a05cb9776084131a562fd3e67e8f3cb, entries=250, sequenceid=133, filesize=16.5 K 2024-11-27T12:24:14,066 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/.tmp/B/62133eb7cc03471aa362cef0dbd6e620 as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/B/62133eb7cc03471aa362cef0dbd6e620 2024-11-27T12:24:14,083 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/B/62133eb7cc03471aa362cef0dbd6e620, entries=150, sequenceid=133, filesize=11.9 K 2024-11-27T12:24:14,086 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/.tmp/C/20ba43d95b3449329b7c56e9754ba8c9 as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/C/20ba43d95b3449329b7c56e9754ba8c9 2024-11-27T12:24:14,091 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-27T12:24:14,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4. 2024-11-27T12:24:14,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4. as already flushing 2024-11-27T12:24:14,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4. 2024-11-27T12:24:14,092 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 953f259f29896493f17802c953dd7fd4, NAME => 'TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:24:14,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 953f259f29896493f17802c953dd7fd4, NAME => 'TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:24:14,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36215 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 953f259f29896493f17802c953dd7fd4, NAME => 'TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 953f259f29896493f17802c953dd7fd4, NAME => 'TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:24:14,098 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/C/20ba43d95b3449329b7c56e9754ba8c9, entries=150, sequenceid=133, filesize=11.9 K 2024-11-27T12:24:14,103 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=114.05 KB/116790 for 953f259f29896493f17802c953dd7fd4 in 697ms, sequenceid=133, compaction requested=true 2024-11-27T12:24:14,103 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 953f259f29896493f17802c953dd7fd4: 2024-11-27T12:24:14,103 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 953f259f29896493f17802c953dd7fd4:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T12:24:14,103 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-27T12:24:14,103 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T12:24:14,103 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 953f259f29896493f17802c953dd7fd4:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T12:24:14,103 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T12:24:14,103 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 953f259f29896493f17802c953dd7fd4:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T12:24:14,103 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-27T12:24:14,104 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-27T12:24:14,106 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 62386 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-27T12:24:14,107 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HStore(1541): 953f259f29896493f17802c953dd7fd4/A is initiating minor compaction (all files) 2024-11-27T12:24:14,107 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 953f259f29896493f17802c953dd7fd4/A in TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4. 2024-11-27T12:24:14,107 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/A/c77e507f29524cd1b12ee7dfed66f019, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/A/68e4df30616f480c975bf82bc491a7bc, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/A/e56e87dcf7ce477f9248e78c39bfd4ac, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/A/6a05cb9776084131a562fd3e67e8f3cb] into tmpdir=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/.tmp, totalSize=60.9 K 2024-11-27T12:24:14,108 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.Compactor(225): Compacting c77e507f29524cd1b12ee7dfed66f019, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1732710249433 2024-11-27T12:24:14,109 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48292 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-27T12:24:14,109 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.Compactor(225): Compacting 68e4df30616f480c975bf82bc491a7bc, keycount=350, bloomtype=ROW, size=20.9 K, encoding=NONE, compression=NONE, seqNum=90, earliestPutTs=1732710250630 2024-11-27T12:24:14,109 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HStore(1541): 953f259f29896493f17802c953dd7fd4/B is initiating minor compaction (all files) 2024-11-27T12:24:14,110 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 953f259f29896493f17802c953dd7fd4/B in TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4. 2024-11-27T12:24:14,110 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/B/a0f5d8167eeb4429b13790a4e760f1bd, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/B/048712a3d3244a8799d60dc5fd1312e8, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/B/2a35b97e787c4035a00a16b41757adc4, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/B/62133eb7cc03471aa362cef0dbd6e620] into tmpdir=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/.tmp, totalSize=47.2 K 2024-11-27T12:24:14,110 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.Compactor(225): Compacting e56e87dcf7ce477f9248e78c39bfd4ac, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1732710251511 2024-11-27T12:24:14,111 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] compactions.Compactor(225): Compacting a0f5d8167eeb4429b13790a4e760f1bd, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1732710249433 2024-11-27T12:24:14,111 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.Compactor(225): Compacting 6a05cb9776084131a562fd3e67e8f3cb, keycount=250, bloomtype=ROW, size=16.5 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1732710252703 2024-11-27T12:24:14,111 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] compactions.Compactor(225): Compacting 048712a3d3244a8799d60dc5fd1312e8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=90, earliestPutTs=1732710250651 2024-11-27T12:24:14,112 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2a35b97e787c4035a00a16b41757adc4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1732710251511 2024-11-27T12:24:14,114 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] compactions.Compactor(225): Compacting 62133eb7cc03471aa362cef0dbd6e620, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1732710253392 2024-11-27T12:24:14,143 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T12:24:14,144 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 953f259f29896493f17802c953dd7fd4 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-27T12:24:14,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(8855): Flush requested on 953f259f29896493f17802c953dd7fd4 2024-11-27T12:24:14,160 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 953f259f29896493f17802c953dd7fd4#B#compaction#24 average throughput is 0.36 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-27T12:24:14,161 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/.tmp/B/4356547e1d5f47b3b1a12539cc3a6a3e is 50, key is test_row_0/B:col10/1732710253395/Put/seqid=0 2024-11-27T12:24:14,170 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/.tmp/A/14c423e1e7914d0f8488ef8648517e1a is 50, key is test_row_0/A:col10/1732710253493/Put/seqid=0 2024-11-27T12:24:14,175 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 953f259f29896493f17802c953dd7fd4#A#compaction#25 average throughput is 0.30 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T12:24:14,176 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/.tmp/A/aca784817a654817a555b448746a7ae8 is 50, key is test_row_0/A:col10/1732710253395/Put/seqid=0 2024-11-27T12:24:14,230 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073741861_1037 (size=12425) 2024-11-27T12:24:14,245 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/.tmp/B/4356547e1d5f47b3b1a12539cc3a6a3e as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/B/4356547e1d5f47b3b1a12539cc3a6a3e 2024-11-27T12:24:14,249 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-27T12:24:14,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4. 2024-11-27T12:24:14,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4. as already flushing 2024-11-27T12:24:14,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4. 2024-11-27T12:24:14,250 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 953f259f29896493f17802c953dd7fd4, NAME => 'TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:24:14,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 953f259f29896493f17802c953dd7fd4, NAME => 'TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:24:14,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-27T12:24:14,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36215 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 953f259f29896493f17802c953dd7fd4, NAME => 'TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 953f259f29896493f17802c953dd7fd4, NAME => 'TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:24:14,261 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073741862_1038 (size=14541) 2024-11-27T12:24:14,262 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=154 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/.tmp/A/14c423e1e7914d0f8488ef8648517e1a 2024-11-27T12:24:14,274 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:14,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60348 deadline: 1732710314262, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:14,277 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:14,277 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 953f259f29896493f17802c953dd7fd4/B of 953f259f29896493f17802c953dd7fd4 into 4356547e1d5f47b3b1a12539cc3a6a3e(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T12:24:14,277 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 953f259f29896493f17802c953dd7fd4: 2024-11-27T12:24:14,277 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., storeName=953f259f29896493f17802c953dd7fd4/B, priority=12, startTime=1732710254103; duration=0sec 2024-11-27T12:24:14,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60298 deadline: 1732710314269, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:14,277 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T12:24:14,277 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 953f259f29896493f17802c953dd7fd4:B 2024-11-27T12:24:14,277 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-27T12:24:14,278 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:14,278 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:14,278 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:24:14,278 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:14,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60300 deadline: 1732710314258, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:14,279 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:14,279 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:14,279 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:14,279 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:14,279 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:24:14,279 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:24:14,282 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073741863_1039 (size=12425) 2024-11-27T12:24:14,290 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48292 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-27T12:24:14,290 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HStore(1541): 953f259f29896493f17802c953dd7fd4/C is initiating minor compaction (all files) 2024-11-27T12:24:14,290 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 953f259f29896493f17802c953dd7fd4/C in TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4. 2024-11-27T12:24:14,290 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/C/e439ae32af0b4328bf72d5106a7e9cbe, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/C/e0864f52a7ca4630b2b957d17bded558, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/C/d047d654ce40415780ab3e93779afec5, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/C/20ba43d95b3449329b7c56e9754ba8c9] into tmpdir=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/.tmp, totalSize=47.2 K 2024-11-27T12:24:14,294 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] compactions.Compactor(225): Compacting e439ae32af0b4328bf72d5106a7e9cbe, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1732710249433 2024-11-27T12:24:14,296 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] compactions.Compactor(225): Compacting e0864f52a7ca4630b2b957d17bded558, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=90, earliestPutTs=1732710250651 2024-11-27T12:24:14,298 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] compactions.Compactor(225): Compacting d047d654ce40415780ab3e93779afec5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1732710251511 2024-11-27T12:24:14,302 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] compactions.Compactor(225): Compacting 20ba43d95b3449329b7c56e9754ba8c9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1732710253392 2024-11-27T12:24:14,305 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/.tmp/A/aca784817a654817a555b448746a7ae8 as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/A/aca784817a654817a555b448746a7ae8 2024-11-27T12:24:14,330 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/.tmp/B/ddc5a2d40cab425f81fc2065b32ee7f2 is 50, key is test_row_0/B:col10/1732710253493/Put/seqid=0 2024-11-27T12:24:14,336 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 953f259f29896493f17802c953dd7fd4/A of 953f259f29896493f17802c953dd7fd4 into aca784817a654817a555b448746a7ae8(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T12:24:14,336 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 953f259f29896493f17802c953dd7fd4: 2024-11-27T12:24:14,336 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., storeName=953f259f29896493f17802c953dd7fd4/A, priority=12, startTime=1732710254103; duration=0sec 2024-11-27T12:24:14,336 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T12:24:14,337 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 953f259f29896493f17802c953dd7fd4:A 2024-11-27T12:24:14,347 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 953f259f29896493f17802c953dd7fd4#C#compaction#28 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T12:24:14,348 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073741864_1040 (size=12151) 2024-11-27T12:24:14,350 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/.tmp/C/3eca934aa97f43cb982022ae17077462 is 50, key is test_row_0/C:col10/1732710253395/Put/seqid=0 2024-11-27T12:24:14,357 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=154 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/.tmp/B/ddc5a2d40cab425f81fc2065b32ee7f2 2024-11-27T12:24:14,394 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:14,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60300 deadline: 1732710314389, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:14,395 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:14,395 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:14,395 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:24:14,395 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:14,396 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:14,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60348 deadline: 1732710314389, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:14,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60298 deadline: 1732710314389, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:14,397 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:14,397 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:14,397 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:14,397 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:14,397 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:24:14,397 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:24:14,397 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/.tmp/C/826694515f524af08e9fa77d290e62d7 is 50, key is test_row_0/C:col10/1732710253493/Put/seqid=0 2024-11-27T12:24:14,397 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073741865_1041 (size=12425) 2024-11-27T12:24:14,407 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-27T12:24:14,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4. 2024-11-27T12:24:14,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4. as already flushing 2024-11-27T12:24:14,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4. 2024-11-27T12:24:14,408 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 953f259f29896493f17802c953dd7fd4, NAME => 'TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:24:14,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 953f259f29896493f17802c953dd7fd4, NAME => 'TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:24:14,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36215 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 953f259f29896493f17802c953dd7fd4, NAME => 'TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 953f259f29896493f17802c953dd7fd4, NAME => 'TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:24:14,424 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/.tmp/C/3eca934aa97f43cb982022ae17077462 as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/C/3eca934aa97f43cb982022ae17077462 2024-11-27T12:24:14,440 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073741866_1042 (size=12151) 2024-11-27T12:24:14,442 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=154 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/.tmp/C/826694515f524af08e9fa77d290e62d7 2024-11-27T12:24:14,450 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 953f259f29896493f17802c953dd7fd4/C of 953f259f29896493f17802c953dd7fd4 into 3eca934aa97f43cb982022ae17077462(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T12:24:14,450 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 953f259f29896493f17802c953dd7fd4: 2024-11-27T12:24:14,450 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., storeName=953f259f29896493f17802c953dd7fd4/C, priority=12, startTime=1732710254103; duration=0sec 2024-11-27T12:24:14,450 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T12:24:14,451 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 953f259f29896493f17802c953dd7fd4:C 2024-11-27T12:24:14,453 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/.tmp/A/14c423e1e7914d0f8488ef8648517e1a as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/A/14c423e1e7914d0f8488ef8648517e1a 2024-11-27T12:24:14,464 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/A/14c423e1e7914d0f8488ef8648517e1a, entries=200, sequenceid=154, filesize=14.2 K 2024-11-27T12:24:14,467 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/.tmp/B/ddc5a2d40cab425f81fc2065b32ee7f2 as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/B/ddc5a2d40cab425f81fc2065b32ee7f2 2024-11-27T12:24:14,482 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/B/ddc5a2d40cab425f81fc2065b32ee7f2, entries=150, sequenceid=154, filesize=11.9 K 2024-11-27T12:24:14,484 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/.tmp/C/826694515f524af08e9fa77d290e62d7 as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/C/826694515f524af08e9fa77d290e62d7 2024-11-27T12:24:14,500 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/C/826694515f524af08e9fa77d290e62d7, entries=150, sequenceid=154, filesize=11.9 K 2024-11-27T12:24:14,503 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=80.51 KB/82440 for 953f259f29896493f17802c953dd7fd4 in 360ms, sequenceid=154, compaction requested=false 2024-11-27T12:24:14,503 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 953f259f29896493f17802c953dd7fd4: 2024-11-27T12:24:14,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-27T12:24:14,566 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-27T12:24:14,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4. 2024-11-27T12:24:14,566 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2902): Flushing 953f259f29896493f17802c953dd7fd4 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-27T12:24:14,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/.tmp/A/4cea0403883b4e6090566caa7b04f5c4 is 50, key is test_row_0/A:col10/1732710254256/Put/seqid=0 2024-11-27T12:24:14,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(8855): Flush requested on 953f259f29896493f17802c953dd7fd4 2024-11-27T12:24:14,620 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4. as already flushing 2024-11-27T12:24:14,626 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073741867_1043 (size=12151) 2024-11-27T12:24:14,627 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=172 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/.tmp/A/4cea0403883b4e6090566caa7b04f5c4 2024-11-27T12:24:14,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/.tmp/B/25d3820f844e41a8ba05baf59f997efc is 50, key is test_row_0/B:col10/1732710254256/Put/seqid=0 2024-11-27T12:24:14,660 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073741868_1044 (size=12151) 2024-11-27T12:24:14,696 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:14,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60348 deadline: 1732710314691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:14,697 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:14,697 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:14,698 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:24:14,700 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:14,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60298 deadline: 1732710314695, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:14,701 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:14,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60300 deadline: 1732710314695, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:14,708 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:14,708 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:14,708 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:24:14,712 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:14,712 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:14,712 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:24:14,814 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:14,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60348 deadline: 1732710314809, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:14,815 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:14,815 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:14,815 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:24:14,822 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:14,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60300 deadline: 1732710314818, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:14,823 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:14,823 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:14,823 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:14,823 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:24:14,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60298 deadline: 1732710314818, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:14,823 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:14,823 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:14,823 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:24:15,024 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:15,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60348 deadline: 1732710315018, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:15,025 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:15,025 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:15,025 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:24:15,030 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:15,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60300 deadline: 1732710315029, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:15,031 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:15,031 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:15,031 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:15,031 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:24:15,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60298 deadline: 1732710315029, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:15,032 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:15,032 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:15,032 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:24:15,062 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=172 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/.tmp/B/25d3820f844e41a8ba05baf59f997efc 2024-11-27T12:24:15,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-27T12:24:15,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/.tmp/C/a272e3ce38ae4891b7b55842fc31aed0 is 50, key is test_row_0/C:col10/1732710254256/Put/seqid=0 2024-11-27T12:24:15,111 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073741869_1045 (size=12151) 2024-11-27T12:24:15,112 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=172 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/.tmp/C/a272e3ce38ae4891b7b55842fc31aed0 2024-11-27T12:24:15,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/.tmp/A/4cea0403883b4e6090566caa7b04f5c4 as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/A/4cea0403883b4e6090566caa7b04f5c4 2024-11-27T12:24:15,142 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/A/4cea0403883b4e6090566caa7b04f5c4, entries=150, sequenceid=172, filesize=11.9 K 2024-11-27T12:24:15,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/.tmp/B/25d3820f844e41a8ba05baf59f997efc as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/B/25d3820f844e41a8ba05baf59f997efc 2024-11-27T12:24:15,162 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/B/25d3820f844e41a8ba05baf59f997efc, entries=150, sequenceid=172, filesize=11.9 K 2024-11-27T12:24:15,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/.tmp/C/a272e3ce38ae4891b7b55842fc31aed0 as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/C/a272e3ce38ae4891b7b55842fc31aed0 2024-11-27T12:24:15,182 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/C/a272e3ce38ae4891b7b55842fc31aed0, entries=150, sequenceid=172, filesize=11.9 K 2024-11-27T12:24:15,185 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for 953f259f29896493f17802c953dd7fd4 in 619ms, sequenceid=172, compaction requested=true 2024-11-27T12:24:15,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2603): Flush status journal for 953f259f29896493f17802c953dd7fd4: 2024-11-27T12:24:15,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4. 2024-11-27T12:24:15,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=12 2024-11-27T12:24:15,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36215 {}] master.HMaster(4169): Remote procedure done, pid=12 2024-11-27T12:24:15,193 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11 2024-11-27T12:24:15,194 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2530 sec 2024-11-27T12:24:15,199 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees in 1.2750 sec 2024-11-27T12:24:15,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(8855): Flush requested on 953f259f29896493f17802c953dd7fd4 2024-11-27T12:24:15,334 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T12:24:15,334 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 953f259f29896493f17802c953dd7fd4 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-27T12:24:15,355 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/.tmp/A/63bd5a4637144099bf2ebad11527a6df is 50, key is test_row_0/A:col10/1732710254649/Put/seqid=0 2024-11-27T12:24:15,379 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:15,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60298 deadline: 1732710315373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:15,380 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:15,380 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:15,380 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:24:15,385 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073741870_1046 (size=14541) 2024-11-27T12:24:15,387 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:15,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60348 deadline: 1732710315377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:15,387 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=194 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/.tmp/A/63bd5a4637144099bf2ebad11527a6df 2024-11-27T12:24:15,388 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:15,388 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:15,388 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:15,388 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:24:15,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60300 deadline: 1732710315379, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:15,389 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:15,389 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:15,389 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:24:15,421 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/.tmp/B/49a8dc03ebad4782a37eafc183a6ead7 is 50, key is test_row_0/B:col10/1732710254649/Put/seqid=0 2024-11-27T12:24:15,455 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073741871_1047 (size=12151) 2024-11-27T12:24:15,490 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:15,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60298 deadline: 1732710315488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:15,492 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:15,492 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:15,492 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:24:15,506 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:15,506 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:15,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60300 deadline: 1732710315501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:15,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60348 deadline: 1732710315501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:15,508 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:15,508 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:15,508 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:15,508 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:15,508 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:24:15,508 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:24:15,700 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:15,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60298 deadline: 1732710315698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:15,701 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:15,701 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:15,701 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:24:15,720 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:15,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60348 deadline: 1732710315718, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:15,721 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:15,721 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:15,721 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:24:15,721 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:15,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60300 deadline: 1732710315718, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:15,722 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:15,722 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:15,722 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:24:15,856 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=194 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/.tmp/B/49a8dc03ebad4782a37eafc183a6ead7 2024-11-27T12:24:15,871 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/.tmp/C/b9112910c84d4698b52b5c5ba3428703 is 50, key is test_row_0/C:col10/1732710254649/Put/seqid=0 2024-11-27T12:24:15,877 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073741872_1048 (size=12151) 2024-11-27T12:24:16,010 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:16,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60298 deadline: 1732710316008, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:16,012 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:16,012 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:16,012 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:24:16,030 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:16,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60348 deadline: 1732710316029, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:16,031 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:16,031 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:16,031 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:16,031 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:24:16,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60300 deadline: 1732710316029, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:16,032 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:16,032 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:16,032 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:24:16,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-27T12:24:16,088 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T12:24:16,089 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36215 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-27T12:24:16,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36215 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees 2024-11-27T12:24:16,096 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T12:24:16,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-27T12:24:16,102 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T12:24:16,102 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T12:24:16,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-27T12:24:16,262 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-27T12:24:16,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4. 2024-11-27T12:24:16,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4. as already flushing 2024-11-27T12:24:16,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4. 2024-11-27T12:24:16,263 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 953f259f29896493f17802c953dd7fd4, NAME => 'TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:24:16,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 953f259f29896493f17802c953dd7fd4, NAME => 'TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:24:16,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36215 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 953f259f29896493f17802c953dd7fd4, NAME => 'TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 953f259f29896493f17802c953dd7fd4, NAME => 'TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:24:16,278 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=194 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/.tmp/C/b9112910c84d4698b52b5c5ba3428703 2024-11-27T12:24:16,296 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/.tmp/A/63bd5a4637144099bf2ebad11527a6df as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/A/63bd5a4637144099bf2ebad11527a6df 2024-11-27T12:24:16,311 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/A/63bd5a4637144099bf2ebad11527a6df, entries=200, sequenceid=194, filesize=14.2 K 2024-11-27T12:24:16,315 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/.tmp/B/49a8dc03ebad4782a37eafc183a6ead7 as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/B/49a8dc03ebad4782a37eafc183a6ead7 2024-11-27T12:24:16,326 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/B/49a8dc03ebad4782a37eafc183a6ead7, entries=150, sequenceid=194, filesize=11.9 K 2024-11-27T12:24:16,327 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/.tmp/C/b9112910c84d4698b52b5c5ba3428703 as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/C/b9112910c84d4698b52b5c5ba3428703 2024-11-27T12:24:16,344 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/C/b9112910c84d4698b52b5c5ba3428703, entries=150, sequenceid=194, filesize=11.9 K 2024-11-27T12:24:16,346 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for 953f259f29896493f17802c953dd7fd4 in 1012ms, sequenceid=194, compaction requested=true 2024-11-27T12:24:16,346 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 953f259f29896493f17802c953dd7fd4: 2024-11-27T12:24:16,346 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 953f259f29896493f17802c953dd7fd4:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T12:24:16,346 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-27T12:24:16,347 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T12:24:16,347 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-27T12:24:16,347 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 953f259f29896493f17802c953dd7fd4:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T12:24:16,347 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T12:24:16,347 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 953f259f29896493f17802c953dd7fd4:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T12:24:16,347 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T12:24:16,349 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 53658 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-27T12:24:16,349 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HStore(1541): 953f259f29896493f17802c953dd7fd4/A is initiating minor compaction (all files) 2024-11-27T12:24:16,350 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 953f259f29896493f17802c953dd7fd4/A in TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4. 2024-11-27T12:24:16,350 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/A/aca784817a654817a555b448746a7ae8, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/A/14c423e1e7914d0f8488ef8648517e1a, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/A/4cea0403883b4e6090566caa7b04f5c4, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/A/63bd5a4637144099bf2ebad11527a6df] into tmpdir=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/.tmp, totalSize=52.4 K 2024-11-27T12:24:16,350 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48878 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-27T12:24:16,350 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HStore(1541): 953f259f29896493f17802c953dd7fd4/B is initiating minor compaction (all files) 2024-11-27T12:24:16,350 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 953f259f29896493f17802c953dd7fd4/B in TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4. 2024-11-27T12:24:16,350 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] compactions.Compactor(225): Compacting aca784817a654817a555b448746a7ae8, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1732710253392 2024-11-27T12:24:16,351 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/B/4356547e1d5f47b3b1a12539cc3a6a3e, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/B/ddc5a2d40cab425f81fc2065b32ee7f2, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/B/25d3820f844e41a8ba05baf59f997efc, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/B/49a8dc03ebad4782a37eafc183a6ead7] into tmpdir=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/.tmp, totalSize=47.7 K 2024-11-27T12:24:16,351 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.Compactor(225): Compacting 4356547e1d5f47b3b1a12539cc3a6a3e, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1732710253392 2024-11-27T12:24:16,351 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] compactions.Compactor(225): Compacting 14c423e1e7914d0f8488ef8648517e1a, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1732710253471 2024-11-27T12:24:16,352 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.Compactor(225): Compacting ddc5a2d40cab425f81fc2065b32ee7f2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1732710253471 2024-11-27T12:24:16,352 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4cea0403883b4e6090566caa7b04f5c4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1732710254240 2024-11-27T12:24:16,352 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.Compactor(225): Compacting 25d3820f844e41a8ba05baf59f997efc, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1732710254240 2024-11-27T12:24:16,352 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] compactions.Compactor(225): Compacting 63bd5a4637144099bf2ebad11527a6df, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1732710254649 2024-11-27T12:24:16,353 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.Compactor(225): Compacting 49a8dc03ebad4782a37eafc183a6ead7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1732710254649 2024-11-27T12:24:16,381 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 953f259f29896493f17802c953dd7fd4#B#compaction#36 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T12:24:16,382 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/.tmp/B/7474690934c34fc8a49810280d442f89 is 50, key is test_row_0/B:col10/1732710254649/Put/seqid=0 2024-11-27T12:24:16,403 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 953f259f29896493f17802c953dd7fd4#A#compaction#37 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T12:24:16,404 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/.tmp/A/d0f468dde04040e0aef7f77be7637e5c is 50, key is test_row_0/A:col10/1732710254649/Put/seqid=0 2024-11-27T12:24:16,418 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-27T12:24:16,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-27T12:24:16,419 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4. 2024-11-27T12:24:16,423 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2902): Flushing 953f259f29896493f17802c953dd7fd4 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-27T12:24:16,430 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073741874_1050 (size=12561) 2024-11-27T12:24:16,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/.tmp/A/26dbdbed0055463dbd2b17335bd10d22 is 50, key is test_row_0/A:col10/1732710255376/Put/seqid=0 2024-11-27T12:24:16,436 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073741873_1049 (size=12561) 2024-11-27T12:24:16,455 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/.tmp/A/d0f468dde04040e0aef7f77be7637e5c as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/A/d0f468dde04040e0aef7f77be7637e5c 2024-11-27T12:24:16,461 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/.tmp/B/7474690934c34fc8a49810280d442f89 as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/B/7474690934c34fc8a49810280d442f89 2024-11-27T12:24:16,461 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073741875_1051 (size=12151) 2024-11-27T12:24:16,463 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=208 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/.tmp/A/26dbdbed0055463dbd2b17335bd10d22 2024-11-27T12:24:16,469 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 953f259f29896493f17802c953dd7fd4/A of 953f259f29896493f17802c953dd7fd4 into d0f468dde04040e0aef7f77be7637e5c(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T12:24:16,469 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 953f259f29896493f17802c953dd7fd4: 2024-11-27T12:24:16,469 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., storeName=953f259f29896493f17802c953dd7fd4/A, priority=12, startTime=1732710256346; duration=0sec 2024-11-27T12:24:16,469 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T12:24:16,469 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 953f259f29896493f17802c953dd7fd4:A 2024-11-27T12:24:16,469 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-27T12:24:16,479 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 953f259f29896493f17802c953dd7fd4/B of 953f259f29896493f17802c953dd7fd4 into 7474690934c34fc8a49810280d442f89(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T12:24:16,479 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 953f259f29896493f17802c953dd7fd4: 2024-11-27T12:24:16,479 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., storeName=953f259f29896493f17802c953dd7fd4/B, priority=12, startTime=1732710256347; duration=0sec 2024-11-27T12:24:16,479 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T12:24:16,479 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 953f259f29896493f17802c953dd7fd4:B 2024-11-27T12:24:16,483 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48878 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-27T12:24:16,483 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HStore(1541): 953f259f29896493f17802c953dd7fd4/C is initiating minor compaction (all files) 2024-11-27T12:24:16,484 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 953f259f29896493f17802c953dd7fd4/C in TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4. 2024-11-27T12:24:16,484 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/C/3eca934aa97f43cb982022ae17077462, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/C/826694515f524af08e9fa77d290e62d7, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/C/a272e3ce38ae4891b7b55842fc31aed0, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/C/b9112910c84d4698b52b5c5ba3428703] into tmpdir=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/.tmp, totalSize=47.7 K 2024-11-27T12:24:16,485 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3eca934aa97f43cb982022ae17077462, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1732710253392 2024-11-27T12:24:16,486 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] compactions.Compactor(225): Compacting 826694515f524af08e9fa77d290e62d7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1732710253471 2024-11-27T12:24:16,487 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] compactions.Compactor(225): Compacting a272e3ce38ae4891b7b55842fc31aed0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1732710254240 2024-11-27T12:24:16,487 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] compactions.Compactor(225): Compacting b9112910c84d4698b52b5c5ba3428703, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1732710254649 2024-11-27T12:24:16,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/.tmp/B/434461a16faa4cf88fa32a5682bb25dd is 50, key is test_row_0/B:col10/1732710255376/Put/seqid=0 2024-11-27T12:24:16,521 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 953f259f29896493f17802c953dd7fd4#C#compaction#40 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T12:24:16,522 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/.tmp/C/f9915d19089f425eaeaeb975bd4396ed is 50, key is test_row_0/C:col10/1732710254649/Put/seqid=0 2024-11-27T12:24:16,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(8855): Flush requested on 953f259f29896493f17802c953dd7fd4 2024-11-27T12:24:16,528 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4. as already flushing 2024-11-27T12:24:16,571 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073741876_1052 (size=12151) 2024-11-27T12:24:16,586 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073741877_1053 (size=12561) 2024-11-27T12:24:16,673 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:16,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60298 deadline: 1732710316657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:16,676 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:16,676 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:16,676 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:24:16,673 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:16,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60348 deadline: 1732710316661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:16,678 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:16,679 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:16,679 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:24:16,683 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:16,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60300 deadline: 1732710316674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:16,684 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:16,685 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:16,685 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:24:16,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-27T12:24:16,779 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:16,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60298 deadline: 1732710316778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:16,782 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:16,782 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:16,782 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:24:16,792 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:16,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60300 deadline: 1732710316789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:16,793 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:16,793 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:16,793 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:16,794 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:24:16,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60348 deadline: 1732710316789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:16,794 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:16,794 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:16,794 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:24:16,972 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=208 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/.tmp/B/434461a16faa4cf88fa32a5682bb25dd 2024-11-27T12:24:16,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/.tmp/C/93eb828966694f95a04383d56b082129 is 50, key is test_row_0/C:col10/1732710255376/Put/seqid=0 2024-11-27T12:24:16,989 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:16,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60298 deadline: 1732710316988, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:16,990 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:16,991 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:16,991 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:24:17,001 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:17,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60348 deadline: 1732710316999, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:17,002 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:17,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60300 deadline: 1732710316999, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:17,002 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:17,003 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:17,003 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:17,003 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:24:17,003 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:17,003 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:24:17,011 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/.tmp/C/f9915d19089f425eaeaeb975bd4396ed as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/C/f9915d19089f425eaeaeb975bd4396ed 2024-11-27T12:24:17,016 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073741878_1054 (size=12151) 2024-11-27T12:24:17,017 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=208 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/.tmp/C/93eb828966694f95a04383d56b082129 2024-11-27T12:24:17,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/.tmp/A/26dbdbed0055463dbd2b17335bd10d22 as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/A/26dbdbed0055463dbd2b17335bd10d22 2024-11-27T12:24:17,025 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 953f259f29896493f17802c953dd7fd4/C of 953f259f29896493f17802c953dd7fd4 into f9915d19089f425eaeaeb975bd4396ed(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T12:24:17,026 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 953f259f29896493f17802c953dd7fd4: 2024-11-27T12:24:17,026 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., storeName=953f259f29896493f17802c953dd7fd4/C, priority=12, startTime=1732710256347; duration=0sec 2024-11-27T12:24:17,026 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T12:24:17,026 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 953f259f29896493f17802c953dd7fd4:C 2024-11-27T12:24:17,034 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/A/26dbdbed0055463dbd2b17335bd10d22, entries=150, sequenceid=208, filesize=11.9 K 2024-11-27T12:24:17,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/.tmp/B/434461a16faa4cf88fa32a5682bb25dd as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/B/434461a16faa4cf88fa32a5682bb25dd 2024-11-27T12:24:17,047 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/B/434461a16faa4cf88fa32a5682bb25dd, entries=150, sequenceid=208, filesize=11.9 K 2024-11-27T12:24:17,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/.tmp/C/93eb828966694f95a04383d56b082129 as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/C/93eb828966694f95a04383d56b082129 2024-11-27T12:24:17,061 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/C/93eb828966694f95a04383d56b082129, entries=150, sequenceid=208, filesize=11.9 K 2024-11-27T12:24:17,063 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for 953f259f29896493f17802c953dd7fd4 in 641ms, sequenceid=208, compaction requested=false 2024-11-27T12:24:17,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2603): Flush status journal for 953f259f29896493f17802c953dd7fd4: 2024-11-27T12:24:17,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4. 2024-11-27T12:24:17,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=14 2024-11-27T12:24:17,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36215 {}] master.HMaster(4169): Remote procedure done, pid=14 2024-11-27T12:24:17,070 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-11-27T12:24:17,071 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 964 msec 2024-11-27T12:24:17,075 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees in 983 msec 2024-11-27T12:24:17,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-27T12:24:17,238 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T12:24:17,239 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36215 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-27T12:24:17,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36215 {}] procedure2.ProcedureExecutor(1139): Stored pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees 2024-11-27T12:24:17,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-27T12:24:17,244 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T12:24:17,245 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T12:24:17,245 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=15, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T12:24:17,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(8855): Flush requested on 953f259f29896493f17802c953dd7fd4 2024-11-27T12:24:17,302 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T12:24:17,302 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 953f259f29896493f17802c953dd7fd4 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-27T12:24:17,319 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/.tmp/A/0290e3cdd8874318b8965ad3e503f629 is 50, key is test_row_0/A:col10/1732710256601/Put/seqid=0 2024-11-27T12:24:17,341 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:17,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60298 deadline: 1732710317334, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:17,342 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:17,341 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:17,342 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:17,342 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:24:17,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60348 deadline: 1732710317334, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:17,342 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:17,342 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:17,342 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:17,342 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:24:17,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60300 deadline: 1732710317337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:17,343 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:17,343 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:17,343 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:24:17,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-27T12:24:17,359 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073741879_1055 (size=12151) 2024-11-27T12:24:17,361 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=234 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/.tmp/A/0290e3cdd8874318b8965ad3e503f629 2024-11-27T12:24:17,379 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/.tmp/B/0c94518a184f4bd3ab886ee201ec40ff is 50, key is test_row_0/B:col10/1732710256601/Put/seqid=0 2024-11-27T12:24:17,386 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073741880_1056 (size=12151) 2024-11-27T12:24:17,398 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-27T12:24:17,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4. 2024-11-27T12:24:17,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4. as already flushing 2024-11-27T12:24:17,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4. 2024-11-27T12:24:17,399 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 953f259f29896493f17802c953dd7fd4, NAME => 'TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:24:17,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 953f259f29896493f17802c953dd7fd4, NAME => 'TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:24:17,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36215 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 953f259f29896493f17802c953dd7fd4, NAME => 'TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 953f259f29896493f17802c953dd7fd4, NAME => 'TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:24:17,450 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:17,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60298 deadline: 1732710317448, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:17,450 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:17,450 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:17,450 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:17,451 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:24:17,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60348 deadline: 1732710317449, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:17,451 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:17,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60300 deadline: 1732710317449, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:17,451 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:17,451 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:17,451 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:24:17,451 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:17,451 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:17,451 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:24:17,552 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-27T12:24:17,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4. 2024-11-27T12:24:17,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4. as already flushing 2024-11-27T12:24:17,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4. 2024-11-27T12:24:17,553 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 953f259f29896493f17802c953dd7fd4, NAME => 'TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:24:17,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 953f259f29896493f17802c953dd7fd4, NAME => 'TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:24:17,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36215 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 953f259f29896493f17802c953dd7fd4, NAME => 'TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 953f259f29896493f17802c953dd7fd4, NAME => 'TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:24:17,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-27T12:24:17,658 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:17,658 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:17,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60298 deadline: 1732710317658, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:17,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60348 deadline: 1732710317658, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:17,659 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:17,659 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:17,659 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:17,659 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:17,659 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:17,659 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:24:17,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60300 deadline: 1732710317658, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:17,659 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:24:17,660 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:17,660 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:17,660 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:24:17,706 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-27T12:24:17,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4. 2024-11-27T12:24:17,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4. as already flushing 2024-11-27T12:24:17,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4. 2024-11-27T12:24:17,707 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 953f259f29896493f17802c953dd7fd4, NAME => 'TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:24:17,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 953f259f29896493f17802c953dd7fd4, NAME => 'TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:24:17,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36215 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 953f259f29896493f17802c953dd7fd4, NAME => 'TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 953f259f29896493f17802c953dd7fd4, NAME => 'TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:24:17,721 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:17,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60292 deadline: 1732710317718, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:17,722 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 1ba5af43fd99,36251,1732710241631 for 'test_row_2' in 953f259f29896493f17802c953dd7fd4 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8246 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T12:24:17,722 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:17,722 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:17,722 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:24:17,732 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:17,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60356 deadline: 1732710317728, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:17,733 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 1ba5af43fd99,36251,1732710241631 for 'test_row_0' in 953f259f29896493f17802c953dd7fd4 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8253 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T12:24:17,733 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:17,733 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:17,733 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:24:17,787 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=234 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/.tmp/B/0c94518a184f4bd3ab886ee201ec40ff 2024-11-27T12:24:17,802 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/.tmp/C/041f7410542a4874816bf4175bebd6fb is 50, key is test_row_0/C:col10/1732710256601/Put/seqid=0 2024-11-27T12:24:17,810 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073741881_1057 (size=12151) 2024-11-27T12:24:17,813 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=234 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/.tmp/C/041f7410542a4874816bf4175bebd6fb 2024-11-27T12:24:17,825 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/.tmp/A/0290e3cdd8874318b8965ad3e503f629 as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/A/0290e3cdd8874318b8965ad3e503f629 2024-11-27T12:24:17,835 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/A/0290e3cdd8874318b8965ad3e503f629, entries=150, sequenceid=234, filesize=11.9 K 2024-11-27T12:24:17,837 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/.tmp/B/0c94518a184f4bd3ab886ee201ec40ff as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/B/0c94518a184f4bd3ab886ee201ec40ff 2024-11-27T12:24:17,847 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/B/0c94518a184f4bd3ab886ee201ec40ff, entries=150, sequenceid=234, filesize=11.9 K 2024-11-27T12:24:17,849 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/.tmp/C/041f7410542a4874816bf4175bebd6fb as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/C/041f7410542a4874816bf4175bebd6fb 2024-11-27T12:24:17,863 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/C/041f7410542a4874816bf4175bebd6fb, entries=150, sequenceid=234, filesize=11.9 K 2024-11-27T12:24:17,866 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-27T12:24:17,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4. 2024-11-27T12:24:17,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4. as already flushing 2024-11-27T12:24:17,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4. 2024-11-27T12:24:17,866 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for 953f259f29896493f17802c953dd7fd4 in 564ms, sequenceid=234, compaction requested=true 2024-11-27T12:24:17,867 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 953f259f29896493f17802c953dd7fd4: 2024-11-27T12:24:17,866 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 953f259f29896493f17802c953dd7fd4, NAME => 'TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:24:17,867 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 953f259f29896493f17802c953dd7fd4:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T12:24:17,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 953f259f29896493f17802c953dd7fd4, NAME => 'TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:24:17,867 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T12:24:17,867 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T12:24:17,867 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 953f259f29896493f17802c953dd7fd4:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T12:24:17,867 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T12:24:17,867 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T12:24:17,867 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 953f259f29896493f17802c953dd7fd4:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T12:24:17,867 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T12:24:17,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36215 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 953f259f29896493f17802c953dd7fd4, NAME => 'TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 953f259f29896493f17802c953dd7fd4, NAME => 'TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:24:17,868 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T12:24:17,868 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HStore(1541): 953f259f29896493f17802c953dd7fd4/B is initiating minor compaction (all files) 2024-11-27T12:24:17,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-27T12:24:17,869 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 953f259f29896493f17802c953dd7fd4/B in TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4. 2024-11-27T12:24:17,869 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/B/7474690934c34fc8a49810280d442f89, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/B/434461a16faa4cf88fa32a5682bb25dd, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/B/0c94518a184f4bd3ab886ee201ec40ff] into tmpdir=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/.tmp, totalSize=36.0 K 2024-11-27T12:24:17,869 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T12:24:17,869 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HStore(1541): 953f259f29896493f17802c953dd7fd4/A is initiating minor compaction (all files) 2024-11-27T12:24:17,869 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 953f259f29896493f17802c953dd7fd4/A in TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4. 2024-11-27T12:24:17,870 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/A/d0f468dde04040e0aef7f77be7637e5c, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/A/26dbdbed0055463dbd2b17335bd10d22, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/A/0290e3cdd8874318b8965ad3e503f629] into tmpdir=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/.tmp, totalSize=36.0 K 2024-11-27T12:24:17,870 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.Compactor(225): Compacting 7474690934c34fc8a49810280d442f89, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1732710254649 2024-11-27T12:24:17,870 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] compactions.Compactor(225): Compacting d0f468dde04040e0aef7f77be7637e5c, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1732710254649 2024-11-27T12:24:17,870 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.Compactor(225): Compacting 434461a16faa4cf88fa32a5682bb25dd, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1732710255365 2024-11-27T12:24:17,871 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.Compactor(225): Compacting 0c94518a184f4bd3ab886ee201ec40ff, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=234, earliestPutTs=1732710256588 2024-11-27T12:24:17,871 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] compactions.Compactor(225): Compacting 26dbdbed0055463dbd2b17335bd10d22, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1732710255365 2024-11-27T12:24:17,872 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0290e3cdd8874318b8965ad3e503f629, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=234, earliestPutTs=1732710256588 2024-11-27T12:24:17,886 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 953f259f29896493f17802c953dd7fd4#B#compaction#45 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T12:24:17,887 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/.tmp/B/c886c2a649564c6fbbc97f98416226d3 is 50, key is test_row_0/B:col10/1732710256601/Put/seqid=0 2024-11-27T12:24:17,890 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 953f259f29896493f17802c953dd7fd4#A#compaction#46 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T12:24:17,890 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/.tmp/A/47295f031e6c4a7cbbf1ce4923cabb48 is 50, key is test_row_0/A:col10/1732710256601/Put/seqid=0 2024-11-27T12:24:17,941 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073741882_1058 (size=12663) 2024-11-27T12:24:17,958 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073741883_1059 (size=12663) 2024-11-27T12:24:17,959 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/.tmp/B/c886c2a649564c6fbbc97f98416226d3 as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/B/c886c2a649564c6fbbc97f98416226d3 2024-11-27T12:24:17,968 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/.tmp/A/47295f031e6c4a7cbbf1ce4923cabb48 as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/A/47295f031e6c4a7cbbf1ce4923cabb48 2024-11-27T12:24:17,971 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 953f259f29896493f17802c953dd7fd4/B of 953f259f29896493f17802c953dd7fd4 into c886c2a649564c6fbbc97f98416226d3(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T12:24:17,971 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 953f259f29896493f17802c953dd7fd4: 2024-11-27T12:24:17,971 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., storeName=953f259f29896493f17802c953dd7fd4/B, priority=13, startTime=1732710257867; duration=0sec 2024-11-27T12:24:17,972 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T12:24:17,972 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 953f259f29896493f17802c953dd7fd4:B 2024-11-27T12:24:17,972 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T12:24:17,974 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T12:24:17,974 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HStore(1541): 953f259f29896493f17802c953dd7fd4/C is initiating minor compaction (all files) 2024-11-27T12:24:17,974 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 953f259f29896493f17802c953dd7fd4/C in TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4. 2024-11-27T12:24:17,974 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/C/f9915d19089f425eaeaeb975bd4396ed, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/C/93eb828966694f95a04383d56b082129, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/C/041f7410542a4874816bf4175bebd6fb] into tmpdir=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/.tmp, totalSize=36.0 K 2024-11-27T12:24:17,975 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T12:24:17,975 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 953f259f29896493f17802c953dd7fd4 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-27T12:24:17,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(8855): Flush requested on 953f259f29896493f17802c953dd7fd4 2024-11-27T12:24:17,975 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.Compactor(225): Compacting f9915d19089f425eaeaeb975bd4396ed, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1732710254649 2024-11-27T12:24:17,976 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.Compactor(225): Compacting 93eb828966694f95a04383d56b082129, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1732710255365 2024-11-27T12:24:17,976 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.Compactor(225): Compacting 041f7410542a4874816bf4175bebd6fb, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=234, earliestPutTs=1732710256588 2024-11-27T12:24:17,980 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 953f259f29896493f17802c953dd7fd4/A of 953f259f29896493f17802c953dd7fd4 into 47295f031e6c4a7cbbf1ce4923cabb48(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T12:24:17,980 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 953f259f29896493f17802c953dd7fd4: 2024-11-27T12:24:17,980 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., storeName=953f259f29896493f17802c953dd7fd4/A, priority=13, startTime=1732710257867; duration=0sec 2024-11-27T12:24:17,980 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T12:24:17,980 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 953f259f29896493f17802c953dd7fd4:A 2024-11-27T12:24:17,994 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/.tmp/A/27c79de79e7b4394b881b28869dab96f is 50, key is test_row_0/A:col10/1732710257973/Put/seqid=0 2024-11-27T12:24:18,014 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 953f259f29896493f17802c953dd7fd4#C#compaction#48 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T12:24:18,015 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/.tmp/C/25dc0f5080c6444c9f9e749c47c5cc7b is 50, key is test_row_0/C:col10/1732710256601/Put/seqid=0 2024-11-27T12:24:18,021 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-27T12:24:18,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4. 2024-11-27T12:24:18,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4. as already flushing 2024-11-27T12:24:18,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4. 2024-11-27T12:24:18,021 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 953f259f29896493f17802c953dd7fd4, NAME => 'TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:24:18,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 953f259f29896493f17802c953dd7fd4, NAME => 'TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:24:18,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36215 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 953f259f29896493f17802c953dd7fd4, NAME => 'TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 953f259f29896493f17802c953dd7fd4, NAME => 'TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:24:18,036 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073741884_1060 (size=12151) 2024-11-27T12:24:18,037 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=251 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/.tmp/A/27c79de79e7b4394b881b28869dab96f 2024-11-27T12:24:18,051 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:18,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60300 deadline: 1732710318038, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:18,052 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:18,052 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:18,052 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:24:18,054 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073741885_1061 (size=12663) 2024-11-27T12:24:18,055 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/.tmp/B/112397a2d108400388bb19842248befb is 50, key is test_row_0/B:col10/1732710257973/Put/seqid=0 2024-11-27T12:24:18,057 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:18,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60298 deadline: 1732710318050, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:18,057 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:18,057 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:18,057 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:18,057 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:24:18,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60348 deadline: 1732710318052, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:18,058 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:18,058 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:18,058 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:24:18,065 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/.tmp/C/25dc0f5080c6444c9f9e749c47c5cc7b as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/C/25dc0f5080c6444c9f9e749c47c5cc7b 2024-11-27T12:24:18,078 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 953f259f29896493f17802c953dd7fd4/C of 953f259f29896493f17802c953dd7fd4 into 25dc0f5080c6444c9f9e749c47c5cc7b(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T12:24:18,078 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 953f259f29896493f17802c953dd7fd4: 2024-11-27T12:24:18,078 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., storeName=953f259f29896493f17802c953dd7fd4/C, priority=13, startTime=1732710257867; duration=0sec 2024-11-27T12:24:18,078 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T12:24:18,078 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 953f259f29896493f17802c953dd7fd4:C 2024-11-27T12:24:18,093 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073741886_1062 (size=12151) 2024-11-27T12:24:18,160 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:18,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60300 deadline: 1732710318158, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:18,161 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:18,161 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:18,161 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:24:18,173 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:18,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60298 deadline: 1732710318168, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:18,173 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:18,174 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:18,173 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:18,174 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:24:18,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60348 deadline: 1732710318169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:18,174 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:18,174 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:18,174 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:24:18,175 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-27T12:24:18,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4. 2024-11-27T12:24:18,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4. as already flushing 2024-11-27T12:24:18,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4. 2024-11-27T12:24:18,176 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 953f259f29896493f17802c953dd7fd4, NAME => 'TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:24:18,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 953f259f29896493f17802c953dd7fd4, NAME => 'TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:24:18,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36215 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 953f259f29896493f17802c953dd7fd4, NAME => 'TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 953f259f29896493f17802c953dd7fd4, NAME => 'TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:24:18,332 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-27T12:24:18,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4. 2024-11-27T12:24:18,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4. as already flushing 2024-11-27T12:24:18,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4. 2024-11-27T12:24:18,332 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 953f259f29896493f17802c953dd7fd4, NAME => 'TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:24:18,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 953f259f29896493f17802c953dd7fd4, NAME => 'TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:24:18,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36215 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 953f259f29896493f17802c953dd7fd4, NAME => 'TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 953f259f29896493f17802c953dd7fd4, NAME => 'TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:24:18,370 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:18,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60300 deadline: 1732710318368, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:18,371 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:18,371 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:18,371 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:24:18,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-27T12:24:18,380 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:18,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60298 deadline: 1732710318378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:18,381 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:18,381 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:18,381 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:18,381 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:24:18,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60348 deadline: 1732710318378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:18,382 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:18,382 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:18,382 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:24:18,489 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-27T12:24:18,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4. 2024-11-27T12:24:18,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4. as already flushing 2024-11-27T12:24:18,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4. 2024-11-27T12:24:18,489 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 953f259f29896493f17802c953dd7fd4, NAME => 'TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:24:18,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 953f259f29896493f17802c953dd7fd4, NAME => 'TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:24:18,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36215 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 953f259f29896493f17802c953dd7fd4, NAME => 'TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 953f259f29896493f17802c953dd7fd4, NAME => 'TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:24:18,494 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=251 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/.tmp/B/112397a2d108400388bb19842248befb 2024-11-27T12:24:18,514 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/.tmp/C/79f9d27b63cb480cadb48c3e55825b7d is 50, key is test_row_0/C:col10/1732710257973/Put/seqid=0 2024-11-27T12:24:18,535 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073741887_1063 (size=12151) 2024-11-27T12:24:18,539 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=251 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/.tmp/C/79f9d27b63cb480cadb48c3e55825b7d 2024-11-27T12:24:18,551 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/.tmp/A/27c79de79e7b4394b881b28869dab96f as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/A/27c79de79e7b4394b881b28869dab96f 2024-11-27T12:24:18,568 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/A/27c79de79e7b4394b881b28869dab96f, entries=150, sequenceid=251, filesize=11.9 K 2024-11-27T12:24:18,570 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/.tmp/B/112397a2d108400388bb19842248befb as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/B/112397a2d108400388bb19842248befb 2024-11-27T12:24:18,597 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/B/112397a2d108400388bb19842248befb, entries=150, sequenceid=251, filesize=11.9 K 2024-11-27T12:24:18,598 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/.tmp/C/79f9d27b63cb480cadb48c3e55825b7d as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/C/79f9d27b63cb480cadb48c3e55825b7d 2024-11-27T12:24:18,605 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/C/79f9d27b63cb480cadb48c3e55825b7d, entries=150, sequenceid=251, filesize=11.9 K 2024-11-27T12:24:18,607 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for 953f259f29896493f17802c953dd7fd4 in 632ms, sequenceid=251, compaction requested=false 2024-11-27T12:24:18,607 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 953f259f29896493f17802c953dd7fd4: 2024-11-27T12:24:18,645 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-27T12:24:18,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4. 2024-11-27T12:24:18,645 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2902): Flushing 953f259f29896493f17802c953dd7fd4 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-27T12:24:18,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/.tmp/A/0c5b478aa7a94bf2a41531c200cb964a is 50, key is test_row_0/A:col10/1732710258049/Put/seqid=0 2024-11-27T12:24:18,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(8855): Flush requested on 953f259f29896493f17802c953dd7fd4 2024-11-27T12:24:18,681 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4. as already flushing 2024-11-27T12:24:18,685 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073741888_1064 (size=12301) 2024-11-27T12:24:18,722 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:18,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60348 deadline: 1732710318713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:18,723 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:18,723 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:18,723 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:24:18,726 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:18,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60298 deadline: 1732710318722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:18,727 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:18,727 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:18,727 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:24:18,728 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:18,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60300 deadline: 1732710318723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:18,729 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:18,729 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:18,729 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:24:18,832 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:18,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60348 deadline: 1732710318828, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:18,833 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:18,833 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:18,833 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:24:18,845 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:18,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60298 deadline: 1732710318838, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:18,846 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:18,845 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:18,846 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:18,846 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:24:18,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60300 deadline: 1732710318838, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:18,846 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:18,846 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:18,846 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:24:19,040 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:19,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60348 deadline: 1732710319038, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:19,041 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:19,042 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:19,042 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:24:19,059 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:19,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60298 deadline: 1732710319058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:19,060 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:19,060 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:19,060 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:19,060 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:24:19,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60300 deadline: 1732710319059, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:19,061 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:19,061 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:19,061 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:24:19,086 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=273 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/.tmp/A/0c5b478aa7a94bf2a41531c200cb964a 2024-11-27T12:24:19,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/.tmp/B/8313896b4eff469fbf570cdc121e3c58 is 50, key is test_row_0/B:col10/1732710258049/Put/seqid=0 2024-11-27T12:24:19,170 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073741889_1065 (size=12301) 2024-11-27T12:24:19,171 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=273 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/.tmp/B/8313896b4eff469fbf570cdc121e3c58 2024-11-27T12:24:19,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/.tmp/C/23b9ea599d97432684675b1826a49ee1 is 50, key is test_row_0/C:col10/1732710258049/Put/seqid=0 2024-11-27T12:24:19,205 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073741890_1066 (size=12301) 2024-11-27T12:24:19,206 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=273 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/.tmp/C/23b9ea599d97432684675b1826a49ee1 2024-11-27T12:24:19,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/.tmp/A/0c5b478aa7a94bf2a41531c200cb964a as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/A/0c5b478aa7a94bf2a41531c200cb964a 2024-11-27T12:24:19,227 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/A/0c5b478aa7a94bf2a41531c200cb964a, entries=150, sequenceid=273, filesize=12.0 K 2024-11-27T12:24:19,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/.tmp/B/8313896b4eff469fbf570cdc121e3c58 as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/B/8313896b4eff469fbf570cdc121e3c58 2024-11-27T12:24:19,240 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/B/8313896b4eff469fbf570cdc121e3c58, entries=150, sequenceid=273, filesize=12.0 K 2024-11-27T12:24:19,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/.tmp/C/23b9ea599d97432684675b1826a49ee1 as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/C/23b9ea599d97432684675b1826a49ee1 2024-11-27T12:24:19,249 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/C/23b9ea599d97432684675b1826a49ee1, entries=150, sequenceid=273, filesize=12.0 K 2024-11-27T12:24:19,251 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=80.51 KB/82440 for 953f259f29896493f17802c953dd7fd4 in 606ms, sequenceid=273, compaction requested=true 2024-11-27T12:24:19,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2603): Flush status journal for 953f259f29896493f17802c953dd7fd4: 2024-11-27T12:24:19,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4. 2024-11-27T12:24:19,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=16 2024-11-27T12:24:19,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36215 {}] master.HMaster(4169): Remote procedure done, pid=16 2024-11-27T12:24:19,258 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=15 2024-11-27T12:24:19,258 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0090 sec 2024-11-27T12:24:19,261 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees in 2.0200 sec 2024-11-27T12:24:19,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(8855): Flush requested on 953f259f29896493f17802c953dd7fd4 2024-11-27T12:24:19,353 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T12:24:19,353 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 953f259f29896493f17802c953dd7fd4 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-27T12:24:19,369 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/.tmp/A/316de21f553f4d0a95a28b20b817f880 is 50, key is test_row_0/A:col10/1732710258711/Put/seqid=0 2024-11-27T12:24:19,378 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073741891_1067 (size=12301) 2024-11-27T12:24:19,378 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=289 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/.tmp/A/316de21f553f4d0a95a28b20b817f880 2024-11-27T12:24:19,387 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/.tmp/B/198c382a6c1b428bbbedd5c3c1fc3ba1 is 50, key is test_row_0/B:col10/1732710258711/Put/seqid=0 2024-11-27T12:24:19,392 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073741892_1068 (size=12301) 2024-11-27T12:24:19,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-27T12:24:19,399 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T12:24:19,400 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36215 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-27T12:24:19,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36215 {}] procedure2.ProcedureExecutor(1139): Stored pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees 2024-11-27T12:24:19,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-27T12:24:19,403 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T12:24:19,404 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T12:24:19,404 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T12:24:19,453 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:19,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60348 deadline: 1732710319447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:19,455 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:19,455 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:19,455 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:24:19,463 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:19,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60298 deadline: 1732710319454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:19,464 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:19,464 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:19,464 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:24:19,464 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:19,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60300 deadline: 1732710319454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:19,464 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:19,465 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:19,465 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:24:19,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-27T12:24:19,557 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-27T12:24:19,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4. 2024-11-27T12:24:19,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4. as already flushing 2024-11-27T12:24:19,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4. 2024-11-27T12:24:19,557 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 953f259f29896493f17802c953dd7fd4, NAME => 'TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:24:19,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 953f259f29896493f17802c953dd7fd4, NAME => 'TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:24:19,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36215 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 953f259f29896493f17802c953dd7fd4, NAME => 'TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 953f259f29896493f17802c953dd7fd4, NAME => 'TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:24:19,561 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:19,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60348 deadline: 1732710319558, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:19,562 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:19,562 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:19,562 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:24:19,568 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:19,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60298 deadline: 1732710319568, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:19,569 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:19,569 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:19,569 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:19,569 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:24:19,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60300 deadline: 1732710319568, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:19,570 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:19,570 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:19,570 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:24:19,711 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-27T12:24:19,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4. 2024-11-27T12:24:19,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4. as already flushing 2024-11-27T12:24:19,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4. 2024-11-27T12:24:19,711 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 953f259f29896493f17802c953dd7fd4, NAME => 'TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:24:19,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 953f259f29896493f17802c953dd7fd4, NAME => 'TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:24:19,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36215 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 953f259f29896493f17802c953dd7fd4, NAME => 'TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 953f259f29896493f17802c953dd7fd4, NAME => 'TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:24:19,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-27T12:24:19,768 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:19,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60348 deadline: 1732710319768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:19,769 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:19,769 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:19,769 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:24:19,780 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:19,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60298 deadline: 1732710319778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:19,781 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:19,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60300 deadline: 1732710319778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:19,782 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:19,782 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:19,782 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:24:19,782 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:19,782 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:19,782 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:24:19,793 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=289 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/.tmp/B/198c382a6c1b428bbbedd5c3c1fc3ba1 2024-11-27T12:24:19,803 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/.tmp/C/d04fdd9a5c054303988cc48465fdd43a is 50, key is test_row_0/C:col10/1732710258711/Put/seqid=0 2024-11-27T12:24:19,808 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073741893_1069 (size=12301) 2024-11-27T12:24:19,865 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-27T12:24:19,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4. 2024-11-27T12:24:19,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4. as already flushing 2024-11-27T12:24:19,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4. 2024-11-27T12:24:19,865 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 953f259f29896493f17802c953dd7fd4, NAME => 'TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:24:19,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 953f259f29896493f17802c953dd7fd4, NAME => 'TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:24:19,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36215 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 953f259f29896493f17802c953dd7fd4, NAME => 'TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 953f259f29896493f17802c953dd7fd4, NAME => 'TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:24:20,019 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-27T12:24:20,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4. 2024-11-27T12:24:20,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4. as already flushing 2024-11-27T12:24:20,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4. 2024-11-27T12:24:20,020 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 953f259f29896493f17802c953dd7fd4, NAME => 'TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:24:20,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 953f259f29896493f17802c953dd7fd4, NAME => 'TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:24:20,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36215 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 953f259f29896493f17802c953dd7fd4, NAME => 'TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 953f259f29896493f17802c953dd7fd4, NAME => 'TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:24:20,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-27T12:24:20,079 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:20,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60348 deadline: 1732710320078, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:20,080 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:20,080 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:20,081 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:24:20,090 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:20,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60300 deadline: 1732710320089, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:20,090 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:20,090 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:20,090 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:24:20,098 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:20,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60298 deadline: 1732710320097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:20,099 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:20,099 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:20,099 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:24:20,172 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-27T12:24:20,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4. 2024-11-27T12:24:20,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4. as already flushing 2024-11-27T12:24:20,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4. 2024-11-27T12:24:20,173 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 953f259f29896493f17802c953dd7fd4, NAME => 'TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:24:20,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 953f259f29896493f17802c953dd7fd4, NAME => 'TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:24:20,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36215 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 953f259f29896493f17802c953dd7fd4, NAME => 'TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 953f259f29896493f17802c953dd7fd4, NAME => 'TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:24:20,209 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=289 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/.tmp/C/d04fdd9a5c054303988cc48465fdd43a 2024-11-27T12:24:20,235 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/.tmp/A/316de21f553f4d0a95a28b20b817f880 as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/A/316de21f553f4d0a95a28b20b817f880 2024-11-27T12:24:20,244 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/A/316de21f553f4d0a95a28b20b817f880, entries=150, sequenceid=289, filesize=12.0 K 2024-11-27T12:24:20,246 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/.tmp/B/198c382a6c1b428bbbedd5c3c1fc3ba1 as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/B/198c382a6c1b428bbbedd5c3c1fc3ba1 2024-11-27T12:24:20,260 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/B/198c382a6c1b428bbbedd5c3c1fc3ba1, entries=150, sequenceid=289, filesize=12.0 K 2024-11-27T12:24:20,262 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/.tmp/C/d04fdd9a5c054303988cc48465fdd43a as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/C/d04fdd9a5c054303988cc48465fdd43a 2024-11-27T12:24:20,271 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/C/d04fdd9a5c054303988cc48465fdd43a, entries=150, sequenceid=289, filesize=12.0 K 2024-11-27T12:24:20,274 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=120.76 KB/123660 for 953f259f29896493f17802c953dd7fd4 in 920ms, sequenceid=289, compaction requested=true 2024-11-27T12:24:20,274 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 953f259f29896493f17802c953dd7fd4: 2024-11-27T12:24:20,274 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 953f259f29896493f17802c953dd7fd4:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T12:24:20,274 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-27T12:24:20,274 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T12:24:20,274 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-27T12:24:20,276 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 953f259f29896493f17802c953dd7fd4:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T12:24:20,277 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49416 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-27T12:24:20,277 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HStore(1541): 953f259f29896493f17802c953dd7fd4/A is initiating minor compaction (all files) 2024-11-27T12:24:20,277 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 953f259f29896493f17802c953dd7fd4/A in TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4. 2024-11-27T12:24:20,278 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/A/47295f031e6c4a7cbbf1ce4923cabb48, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/A/27c79de79e7b4394b881b28869dab96f, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/A/0c5b478aa7a94bf2a41531c200cb964a, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/A/316de21f553f4d0a95a28b20b817f880] into tmpdir=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/.tmp, totalSize=48.3 K 2024-11-27T12:24:20,278 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] compactions.Compactor(225): Compacting 47295f031e6c4a7cbbf1ce4923cabb48, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=234, earliestPutTs=1732710256588 2024-11-27T12:24:20,279 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49416 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-27T12:24:20,279 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HStore(1541): 953f259f29896493f17802c953dd7fd4/B is initiating minor compaction (all files) 2024-11-27T12:24:20,279 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 953f259f29896493f17802c953dd7fd4/B in TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4. 2024-11-27T12:24:20,279 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/B/c886c2a649564c6fbbc97f98416226d3, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/B/112397a2d108400388bb19842248befb, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/B/8313896b4eff469fbf570cdc121e3c58, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/B/198c382a6c1b428bbbedd5c3c1fc3ba1] into tmpdir=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/.tmp, totalSize=48.3 K 2024-11-27T12:24:20,280 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] compactions.Compactor(225): Compacting 27c79de79e7b4394b881b28869dab96f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=251, earliestPutTs=1732710257326 2024-11-27T12:24:20,282 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.Compactor(225): Compacting c886c2a649564c6fbbc97f98416226d3, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=234, earliestPutTs=1732710256588 2024-11-27T12:24:20,283 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0c5b478aa7a94bf2a41531c200cb964a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=273, earliestPutTs=1732710258036 2024-11-27T12:24:20,285 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.Compactor(225): Compacting 112397a2d108400388bb19842248befb, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=251, earliestPutTs=1732710257326 2024-11-27T12:24:20,285 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] compactions.Compactor(225): Compacting 316de21f553f4d0a95a28b20b817f880, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=289, earliestPutTs=1732710258711 2024-11-27T12:24:20,286 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.Compactor(225): Compacting 8313896b4eff469fbf570cdc121e3c58, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=273, earliestPutTs=1732710258036 2024-11-27T12:24:20,287 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.Compactor(225): Compacting 198c382a6c1b428bbbedd5c3c1fc3ba1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=289, earliestPutTs=1732710258711 2024-11-27T12:24:20,288 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T12:24:20,288 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 953f259f29896493f17802c953dd7fd4:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T12:24:20,288 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T12:24:20,314 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 953f259f29896493f17802c953dd7fd4#B#compaction#57 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T12:24:20,314 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/.tmp/B/5a9bb0d6c9654e21ac382e3ce031e732 is 50, key is test_row_0/B:col10/1732710258711/Put/seqid=0 2024-11-27T12:24:20,319 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 953f259f29896493f17802c953dd7fd4#A#compaction#58 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T12:24:20,320 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/.tmp/A/872c35d628a040e2afa1f088e7118972 is 50, key is test_row_0/A:col10/1732710258711/Put/seqid=0 2024-11-27T12:24:20,327 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-27T12:24:20,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4. 2024-11-27T12:24:20,327 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2902): Flushing 953f259f29896493f17802c953dd7fd4 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-27T12:24:20,346 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073741894_1070 (size=12949) 2024-11-27T12:24:20,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/.tmp/A/e46a93be90de468bb732984bda76f025 is 50, key is test_row_0/A:col10/1732710259441/Put/seqid=0 2024-11-27T12:24:20,375 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073741896_1072 (size=12301) 2024-11-27T12:24:20,384 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073741895_1071 (size=12949) 2024-11-27T12:24:20,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-27T12:24:20,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(8855): Flush requested on 953f259f29896493f17802c953dd7fd4 2024-11-27T12:24:20,598 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4. as already flushing 2024-11-27T12:24:20,715 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:20,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60298 deadline: 1732710320709, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:20,715 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:20,716 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:20,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60348 deadline: 1732710320710, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:20,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60300 deadline: 1732710320711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:20,716 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:20,716 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:20,716 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:20,716 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:20,716 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:20,716 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:20,716 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:24:20,716 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:24:20,716 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:24:20,755 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/.tmp/B/5a9bb0d6c9654e21ac382e3ce031e732 as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/B/5a9bb0d6c9654e21ac382e3ce031e732 2024-11-27T12:24:20,764 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 953f259f29896493f17802c953dd7fd4/B of 953f259f29896493f17802c953dd7fd4 into 5a9bb0d6c9654e21ac382e3ce031e732(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T12:24:20,765 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 953f259f29896493f17802c953dd7fd4: 2024-11-27T12:24:20,765 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., storeName=953f259f29896493f17802c953dd7fd4/B, priority=12, startTime=1732710260274; duration=0sec 2024-11-27T12:24:20,765 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T12:24:20,765 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 953f259f29896493f17802c953dd7fd4:B 2024-11-27T12:24:20,765 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-27T12:24:20,769 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49416 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-27T12:24:20,770 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HStore(1541): 953f259f29896493f17802c953dd7fd4/C is initiating minor compaction (all files) 2024-11-27T12:24:20,770 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 953f259f29896493f17802c953dd7fd4/C in TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4. 2024-11-27T12:24:20,770 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/C/25dc0f5080c6444c9f9e749c47c5cc7b, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/C/79f9d27b63cb480cadb48c3e55825b7d, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/C/23b9ea599d97432684675b1826a49ee1, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/C/d04fdd9a5c054303988cc48465fdd43a] into tmpdir=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/.tmp, totalSize=48.3 K 2024-11-27T12:24:20,771 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.Compactor(225): Compacting 25dc0f5080c6444c9f9e749c47c5cc7b, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=234, earliestPutTs=1732710256588 2024-11-27T12:24:20,772 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.Compactor(225): Compacting 79f9d27b63cb480cadb48c3e55825b7d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=251, earliestPutTs=1732710257326 2024-11-27T12:24:20,773 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.Compactor(225): Compacting 23b9ea599d97432684675b1826a49ee1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=273, earliestPutTs=1732710258036 2024-11-27T12:24:20,776 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.Compactor(225): Compacting d04fdd9a5c054303988cc48465fdd43a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=289, earliestPutTs=1732710258711 2024-11-27T12:24:20,777 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=310 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/.tmp/A/e46a93be90de468bb732984bda76f025 2024-11-27T12:24:20,792 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/.tmp/A/872c35d628a040e2afa1f088e7118972 as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/A/872c35d628a040e2afa1f088e7118972 2024-11-27T12:24:20,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/.tmp/B/3393a10bd3e84fd09c57b1570dc60cc1 is 50, key is test_row_0/B:col10/1732710259441/Put/seqid=0 2024-11-27T12:24:20,806 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 953f259f29896493f17802c953dd7fd4/A of 953f259f29896493f17802c953dd7fd4 into 872c35d628a040e2afa1f088e7118972(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T12:24:20,806 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 953f259f29896493f17802c953dd7fd4: 2024-11-27T12:24:20,806 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., storeName=953f259f29896493f17802c953dd7fd4/A, priority=12, startTime=1732710260274; duration=0sec 2024-11-27T12:24:20,806 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T12:24:20,806 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 953f259f29896493f17802c953dd7fd4:A 2024-11-27T12:24:20,811 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 953f259f29896493f17802c953dd7fd4#C#compaction#61 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T12:24:20,811 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/.tmp/C/eedf2d8af6904ff6a7700d253761aaa5 is 50, key is test_row_0/C:col10/1732710258711/Put/seqid=0 2024-11-27T12:24:20,818 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:20,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60348 deadline: 1732710320818, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:20,819 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:20,819 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:20,819 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:24:20,819 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:20,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60300 deadline: 1732710320818, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:20,820 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:20,820 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:20,820 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:24:20,829 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:20,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60298 deadline: 1732710320828, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:20,830 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:20,830 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:20,830 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:24:20,838 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073741898_1074 (size=12949) 2024-11-27T12:24:20,852 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/.tmp/C/eedf2d8af6904ff6a7700d253761aaa5 as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/C/eedf2d8af6904ff6a7700d253761aaa5 2024-11-27T12:24:20,866 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 953f259f29896493f17802c953dd7fd4/C of 953f259f29896493f17802c953dd7fd4 into eedf2d8af6904ff6a7700d253761aaa5(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T12:24:20,867 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 953f259f29896493f17802c953dd7fd4: 2024-11-27T12:24:20,867 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., storeName=953f259f29896493f17802c953dd7fd4/C, priority=12, startTime=1732710260288; duration=0sec 2024-11-27T12:24:20,867 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T12:24:20,867 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 953f259f29896493f17802c953dd7fd4:C 2024-11-27T12:24:20,870 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073741897_1073 (size=12301) 2024-11-27T12:24:21,028 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:21,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60348 deadline: 1732710321028, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:21,029 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:21,029 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:21,029 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:21,029 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:24:21,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60300 deadline: 1732710321028, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:21,029 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:21,029 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:21,029 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:24:21,038 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:21,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60298 deadline: 1732710321038, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:21,039 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:21,039 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:21,039 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:24:21,271 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=310 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/.tmp/B/3393a10bd3e84fd09c57b1570dc60cc1 2024-11-27T12:24:21,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/.tmp/C/853981e7bd1c42d28479f3287a5718d4 is 50, key is test_row_0/C:col10/1732710259441/Put/seqid=0 2024-11-27T12:24:21,305 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073741899_1075 (size=12301) 2024-11-27T12:24:21,306 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=310 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/.tmp/C/853981e7bd1c42d28479f3287a5718d4 2024-11-27T12:24:21,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/.tmp/A/e46a93be90de468bb732984bda76f025 as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/A/e46a93be90de468bb732984bda76f025 2024-11-27T12:24:21,340 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/A/e46a93be90de468bb732984bda76f025, entries=150, sequenceid=310, filesize=12.0 K 2024-11-27T12:24:21,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/.tmp/B/3393a10bd3e84fd09c57b1570dc60cc1 as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/B/3393a10bd3e84fd09c57b1570dc60cc1 2024-11-27T12:24:21,343 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:21,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60348 deadline: 1732710321339, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:21,344 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:21,344 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:21,344 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:24:21,344 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:21,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60300 deadline: 1732710321339, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:21,344 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:21,345 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:21,345 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:24:21,350 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/B/3393a10bd3e84fd09c57b1570dc60cc1, entries=150, sequenceid=310, filesize=12.0 K 2024-11-27T12:24:21,350 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:21,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60298 deadline: 1732710321348, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:21,351 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:21,351 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:21,351 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:24:21,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/.tmp/C/853981e7bd1c42d28479f3287a5718d4 as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/C/853981e7bd1c42d28479f3287a5718d4 2024-11-27T12:24:21,361 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/C/853981e7bd1c42d28479f3287a5718d4, entries=150, sequenceid=310, filesize=12.0 K 2024-11-27T12:24:21,362 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=87.22 KB/89310 for 953f259f29896493f17802c953dd7fd4 in 1035ms, sequenceid=310, compaction requested=false 2024-11-27T12:24:21,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2603): Flush status journal for 953f259f29896493f17802c953dd7fd4: 2024-11-27T12:24:21,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4. 2024-11-27T12:24:21,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=18 2024-11-27T12:24:21,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36215 {}] master.HMaster(4169): Remote procedure done, pid=18 2024-11-27T12:24:21,367 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=17 2024-11-27T12:24:21,368 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9620 sec 2024-11-27T12:24:21,370 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees in 1.9680 sec 2024-11-27T12:24:21,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-27T12:24:21,548 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T12:24:21,549 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36215 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-27T12:24:21,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36215 {}] procedure2.ProcedureExecutor(1139): Stored pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees 2024-11-27T12:24:21,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-27T12:24:21,551 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T12:24:21,553 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T12:24:21,554 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T12:24:21,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-27T12:24:21,707 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-27T12:24:21,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4. 2024-11-27T12:24:21,708 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2902): Flushing 953f259f29896493f17802c953dd7fd4 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-27T12:24:21,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/.tmp/A/5c326aa08229460cae619c871d8137ff is 50, key is test_row_0/A:col10/1732710260709/Put/seqid=0 2024-11-27T12:24:21,724 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073741900_1076 (size=12301) 2024-11-27T12:24:21,724 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=329 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/.tmp/A/5c326aa08229460cae619c871d8137ff 2024-11-27T12:24:21,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/.tmp/B/e4096f31d0df4eac93c447ad55abc18f is 50, key is test_row_0/B:col10/1732710260709/Put/seqid=0 2024-11-27T12:24:21,771 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073741901_1077 (size=12301) 2024-11-27T12:24:21,852 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4. as already flushing 2024-11-27T12:24:21,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(8855): Flush requested on 953f259f29896493f17802c953dd7fd4 2024-11-27T12:24:21,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-27T12:24:21,892 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:21,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60348 deadline: 1732710321885, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:21,893 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:21,893 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:21,893 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:21,893 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:24:21,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60298 deadline: 1732710321885, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:21,894 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:21,894 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:21,894 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:24:21,894 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:21,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60300 deadline: 1732710321888, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:21,897 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:21,897 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:21,897 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:24:22,000 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:22,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60298 deadline: 1732710321998, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:22,001 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:22,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60348 deadline: 1732710321998, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:22,002 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:22,002 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:22,002 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:24:22,002 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:22,003 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:22,003 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:24:22,011 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:22,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60300 deadline: 1732710322007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:22,012 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:22,012 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:22,012 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:24:22,172 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=329 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/.tmp/B/e4096f31d0df4eac93c447ad55abc18f 2024-11-27T12:24:22,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-27T12:24:22,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/.tmp/C/6950ed486d28459c8a25da07153e51a3 is 50, key is test_row_0/C:col10/1732710260709/Put/seqid=0 2024-11-27T12:24:22,208 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:22,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60298 deadline: 1732710322208, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:22,209 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:22,209 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:22,209 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:22,209 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:24:22,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60348 deadline: 1732710322208, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:22,209 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:22,209 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:22,209 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:24:22,218 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073741902_1078 (size=12301) 2024-11-27T12:24:22,221 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:22,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60300 deadline: 1732710322218, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:22,221 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:22,221 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:22,222 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:24:22,525 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:22,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60348 deadline: 1732710322524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:22,526 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:22,526 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:22,526 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:24:22,526 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:22,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60298 deadline: 1732710322524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:22,527 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:22,527 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:22,527 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:24:22,530 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:22,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60300 deadline: 1732710322530, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:22,531 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:22,531 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:22,531 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:24:22,621 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=329 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/.tmp/C/6950ed486d28459c8a25da07153e51a3 2024-11-27T12:24:22,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/.tmp/A/5c326aa08229460cae619c871d8137ff as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/A/5c326aa08229460cae619c871d8137ff 2024-11-27T12:24:22,656 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/A/5c326aa08229460cae619c871d8137ff, entries=150, sequenceid=329, filesize=12.0 K 2024-11-27T12:24:22,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/.tmp/B/e4096f31d0df4eac93c447ad55abc18f as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/B/e4096f31d0df4eac93c447ad55abc18f 2024-11-27T12:24:22,688 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/B/e4096f31d0df4eac93c447ad55abc18f, entries=150, sequenceid=329, filesize=12.0 K 2024-11-27T12:24:22,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-27T12:24:22,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/.tmp/C/6950ed486d28459c8a25da07153e51a3 as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/C/6950ed486d28459c8a25da07153e51a3 2024-11-27T12:24:22,701 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/C/6950ed486d28459c8a25da07153e51a3, entries=150, sequenceid=329, filesize=12.0 K 2024-11-27T12:24:22,702 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=114.05 KB/116790 for 953f259f29896493f17802c953dd7fd4 in 995ms, sequenceid=329, compaction requested=true 2024-11-27T12:24:22,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2603): Flush status journal for 953f259f29896493f17802c953dd7fd4: 2024-11-27T12:24:22,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4. 2024-11-27T12:24:22,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=20 2024-11-27T12:24:22,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36215 {}] master.HMaster(4169): Remote procedure done, pid=20 2024-11-27T12:24:22,707 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=19 2024-11-27T12:24:22,707 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1510 sec 2024-11-27T12:24:22,711 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees in 1.1600 sec 2024-11-27T12:24:23,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(8855): Flush requested on 953f259f29896493f17802c953dd7fd4 2024-11-27T12:24:23,041 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T12:24:23,041 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 953f259f29896493f17802c953dd7fd4 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-27T12:24:23,054 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/.tmp/A/4a0b78d52e164759b66147d4beae2173 is 50, key is test_row_0/A:col10/1732710263041/Put/seqid=0 2024-11-27T12:24:23,087 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:23,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60298 deadline: 1732710323081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:23,087 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:23,087 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:23,086 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:23,088 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:23,088 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:24:23,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60348 deadline: 1732710323083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:23,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60300 deadline: 1732710323079, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:23,088 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:23,088 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:23,088 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:23,088 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:23,088 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:24:23,089 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:24:23,090 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073741903_1079 (size=14741) 2024-11-27T12:24:23,091 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=351 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/.tmp/A/4a0b78d52e164759b66147d4beae2173 2024-11-27T12:24:23,112 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/.tmp/B/fc4015eab12549d7a02f15cf829b47c8 is 50, key is test_row_0/B:col10/1732710263041/Put/seqid=0 2024-11-27T12:24:23,158 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073741904_1080 (size=12301) 2024-11-27T12:24:23,200 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:23,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60300 deadline: 1732710323198, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:23,201 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:23,201 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:23,201 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:23,201 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:24:23,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60298 deadline: 1732710323198, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:23,202 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:23,202 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:23,202 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:23,202 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:24:23,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60348 deadline: 1732710323199, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:23,204 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:23,204 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:23,204 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:24:23,410 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:23,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60300 deadline: 1732710323408, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:23,411 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:23,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60348 deadline: 1732710323408, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:23,411 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:23,411 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:23,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60298 deadline: 1732710323409, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:23,411 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:23,412 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:24:23,412 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:23,412 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:23,412 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:24:23,412 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:23,412 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:23,412 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:24:23,558 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=351 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/.tmp/B/fc4015eab12549d7a02f15cf829b47c8 2024-11-27T12:24:23,571 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/.tmp/C/5a4c44a3b52f4c21a8fb34bd6de79ace is 50, key is test_row_0/C:col10/1732710263041/Put/seqid=0 2024-11-27T12:24:23,580 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073741905_1081 (size=12301) 2024-11-27T12:24:23,582 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=351 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/.tmp/C/5a4c44a3b52f4c21a8fb34bd6de79ace 2024-11-27T12:24:23,593 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/.tmp/A/4a0b78d52e164759b66147d4beae2173 as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/A/4a0b78d52e164759b66147d4beae2173 2024-11-27T12:24:23,600 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/A/4a0b78d52e164759b66147d4beae2173, entries=200, sequenceid=351, filesize=14.4 K 2024-11-27T12:24:23,601 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/.tmp/B/fc4015eab12549d7a02f15cf829b47c8 as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/B/fc4015eab12549d7a02f15cf829b47c8 2024-11-27T12:24:23,608 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/B/fc4015eab12549d7a02f15cf829b47c8, entries=150, sequenceid=351, filesize=12.0 K 2024-11-27T12:24:23,609 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/.tmp/C/5a4c44a3b52f4c21a8fb34bd6de79ace as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/C/5a4c44a3b52f4c21a8fb34bd6de79ace 2024-11-27T12:24:23,620 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/C/5a4c44a3b52f4c21a8fb34bd6de79ace, entries=150, sequenceid=351, filesize=12.0 K 2024-11-27T12:24:23,621 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for 953f259f29896493f17802c953dd7fd4 in 579ms, sequenceid=351, compaction requested=true 2024-11-27T12:24:23,621 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 953f259f29896493f17802c953dd7fd4: 2024-11-27T12:24:23,621 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 953f259f29896493f17802c953dd7fd4:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T12:24:23,621 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T12:24:23,621 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-27T12:24:23,621 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-27T12:24:23,621 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 953f259f29896493f17802c953dd7fd4:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T12:24:23,621 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T12:24:23,621 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 953f259f29896493f17802c953dd7fd4:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T12:24:23,621 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T12:24:23,623 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49852 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-27T12:24:23,623 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HStore(1541): 953f259f29896493f17802c953dd7fd4/B is initiating minor compaction (all files) 2024-11-27T12:24:23,623 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 953f259f29896493f17802c953dd7fd4/B in TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4. 2024-11-27T12:24:23,623 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/B/5a9bb0d6c9654e21ac382e3ce031e732, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/B/3393a10bd3e84fd09c57b1570dc60cc1, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/B/e4096f31d0df4eac93c447ad55abc18f, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/B/fc4015eab12549d7a02f15cf829b47c8] into tmpdir=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/.tmp, totalSize=48.7 K 2024-11-27T12:24:23,624 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.Compactor(225): Compacting 5a9bb0d6c9654e21ac382e3ce031e732, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=289, earliestPutTs=1732710258711 2024-11-27T12:24:23,625 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.Compactor(225): Compacting 3393a10bd3e84fd09c57b1570dc60cc1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=310, earliestPutTs=1732710259439 2024-11-27T12:24:23,625 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 52292 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-27T12:24:23,625 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HStore(1541): 953f259f29896493f17802c953dd7fd4/A is initiating minor compaction (all files) 2024-11-27T12:24:23,625 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 953f259f29896493f17802c953dd7fd4/A in TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4. 2024-11-27T12:24:23,626 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/A/872c35d628a040e2afa1f088e7118972, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/A/e46a93be90de468bb732984bda76f025, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/A/5c326aa08229460cae619c871d8137ff, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/A/4a0b78d52e164759b66147d4beae2173] into tmpdir=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/.tmp, totalSize=51.1 K 2024-11-27T12:24:23,626 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.Compactor(225): Compacting e4096f31d0df4eac93c447ad55abc18f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=329, earliestPutTs=1732710260705 2024-11-27T12:24:23,627 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] compactions.Compactor(225): Compacting 872c35d628a040e2afa1f088e7118972, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=289, earliestPutTs=1732710258711 2024-11-27T12:24:23,627 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.Compactor(225): Compacting fc4015eab12549d7a02f15cf829b47c8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=351, earliestPutTs=1732710261883 2024-11-27T12:24:23,627 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] compactions.Compactor(225): Compacting e46a93be90de468bb732984bda76f025, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=310, earliestPutTs=1732710259439 2024-11-27T12:24:23,628 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5c326aa08229460cae619c871d8137ff, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=329, earliestPutTs=1732710260705 2024-11-27T12:24:23,629 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4a0b78d52e164759b66147d4beae2173, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=351, earliestPutTs=1732710261877 2024-11-27T12:24:23,652 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 953f259f29896493f17802c953dd7fd4#B#compaction#69 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T12:24:23,652 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/.tmp/B/1fe1487421454a07829081b7d1514263 is 50, key is test_row_0/B:col10/1732710263041/Put/seqid=0 2024-11-27T12:24:23,667 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 953f259f29896493f17802c953dd7fd4#A#compaction#70 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T12:24:23,668 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/.tmp/A/11ab16b5c47847e29d71d7be88821602 is 50, key is test_row_0/A:col10/1732710263041/Put/seqid=0 2024-11-27T12:24:23,688 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073741906_1082 (size=13085) 2024-11-27T12:24:23,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-27T12:24:23,698 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T12:24:23,699 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36215 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-27T12:24:23,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36215 {}] procedure2.ProcedureExecutor(1139): Stored pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees 2024-11-27T12:24:23,703 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T12:24:23,704 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T12:24:23,704 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=22, ppid=21, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T12:24:23,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-27T12:24:23,711 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073741907_1083 (size=13085) 2024-11-27T12:24:23,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(8855): Flush requested on 953f259f29896493f17802c953dd7fd4 2024-11-27T12:24:23,721 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T12:24:23,721 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 953f259f29896493f17802c953dd7fd4 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-27T12:24:23,730 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/.tmp/A/36125348ec3f438fb20be16e467bbe30 is 50, key is test_row_0/A:col10/1732710263719/Put/seqid=0 2024-11-27T12:24:23,762 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073741908_1084 (size=17181) 2024-11-27T12:24:23,764 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=367 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/.tmp/A/36125348ec3f438fb20be16e467bbe30 2024-11-27T12:24:23,786 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/.tmp/B/2d7170a3422e4723a86e3315eb291250 is 50, key is test_row_0/B:col10/1732710263719/Put/seqid=0 2024-11-27T12:24:23,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-27T12:24:23,814 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073741909_1085 (size=12301) 2024-11-27T12:24:23,830 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:23,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60348 deadline: 1732710323821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:23,831 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:23,830 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:23,831 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:23,831 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:24:23,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60298 deadline: 1732710323825, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:23,831 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:23,831 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:23,831 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:24:23,836 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:23,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60300 deadline: 1732710323830, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:23,837 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:23,837 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:23,837 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:24:23,859 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-27T12:24:23,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4. 2024-11-27T12:24:23,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4. as already flushing 2024-11-27T12:24:23,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4. 2024-11-27T12:24:23,859 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 953f259f29896493f17802c953dd7fd4, NAME => 'TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:24:23,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 953f259f29896493f17802c953dd7fd4, NAME => 'TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:24:23,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36215 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 953f259f29896493f17802c953dd7fd4, NAME => 'TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 953f259f29896493f17802c953dd7fd4, NAME => 'TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:24:23,942 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:23,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60348 deadline: 1732710323938, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:23,943 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:23,943 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:23,943 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:24:23,944 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:23,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60298 deadline: 1732710323938, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:23,945 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:23,945 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:23,945 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:24:23,949 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:23,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60300 deadline: 1732710323948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:23,950 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:23,950 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:23,951 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:24:24,013 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-27T12:24:24,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4. 2024-11-27T12:24:24,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4. as already flushing 2024-11-27T12:24:24,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4. 2024-11-27T12:24:24,013 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 953f259f29896493f17802c953dd7fd4, NAME => 'TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:24:24,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 953f259f29896493f17802c953dd7fd4, NAME => 'TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:24:24,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36215 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 953f259f29896493f17802c953dd7fd4, NAME => 'TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 953f259f29896493f17802c953dd7fd4, NAME => 'TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:24:24,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-27T12:24:24,100 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/.tmp/B/1fe1487421454a07829081b7d1514263 as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/B/1fe1487421454a07829081b7d1514263 2024-11-27T12:24:24,113 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 953f259f29896493f17802c953dd7fd4/B of 953f259f29896493f17802c953dd7fd4 into 1fe1487421454a07829081b7d1514263(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T12:24:24,113 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 953f259f29896493f17802c953dd7fd4: 2024-11-27T12:24:24,113 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., storeName=953f259f29896493f17802c953dd7fd4/B, priority=12, startTime=1732710263621; duration=0sec 2024-11-27T12:24:24,114 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T12:24:24,114 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 953f259f29896493f17802c953dd7fd4:B 2024-11-27T12:24:24,114 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-27T12:24:24,118 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49852 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-27T12:24:24,118 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HStore(1541): 953f259f29896493f17802c953dd7fd4/C is initiating minor compaction (all files) 2024-11-27T12:24:24,118 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 953f259f29896493f17802c953dd7fd4/C in TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4. 2024-11-27T12:24:24,118 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/C/eedf2d8af6904ff6a7700d253761aaa5, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/C/853981e7bd1c42d28479f3287a5718d4, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/C/6950ed486d28459c8a25da07153e51a3, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/C/5a4c44a3b52f4c21a8fb34bd6de79ace] into tmpdir=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/.tmp, totalSize=48.7 K 2024-11-27T12:24:24,119 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.Compactor(225): Compacting eedf2d8af6904ff6a7700d253761aaa5, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=289, earliestPutTs=1732710258711 2024-11-27T12:24:24,120 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.Compactor(225): Compacting 853981e7bd1c42d28479f3287a5718d4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=310, earliestPutTs=1732710259439 2024-11-27T12:24:24,122 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.Compactor(225): Compacting 6950ed486d28459c8a25da07153e51a3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=329, earliestPutTs=1732710260705 2024-11-27T12:24:24,125 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.Compactor(225): Compacting 5a4c44a3b52f4c21a8fb34bd6de79ace, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=351, earliestPutTs=1732710261883 2024-11-27T12:24:24,126 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/.tmp/A/11ab16b5c47847e29d71d7be88821602 as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/A/11ab16b5c47847e29d71d7be88821602 2024-11-27T12:24:24,146 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 953f259f29896493f17802c953dd7fd4/A of 953f259f29896493f17802c953dd7fd4 into 11ab16b5c47847e29d71d7be88821602(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T12:24:24,146 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 953f259f29896493f17802c953dd7fd4: 2024-11-27T12:24:24,147 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., storeName=953f259f29896493f17802c953dd7fd4/A, priority=12, startTime=1732710263621; duration=0sec 2024-11-27T12:24:24,147 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T12:24:24,147 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 953f259f29896493f17802c953dd7fd4:A 2024-11-27T12:24:24,150 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 953f259f29896493f17802c953dd7fd4#C#compaction#73 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T12:24:24,151 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/.tmp/C/52ac39f85a304ccfa21c8d751673343d is 50, key is test_row_0/C:col10/1732710263041/Put/seqid=0 2024-11-27T12:24:24,153 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:24,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60348 deadline: 1732710324149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:24,153 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:24,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60298 deadline: 1732710324149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:24,154 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:24,154 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:24,154 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:24:24,155 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:24,155 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:24,155 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:24:24,162 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:24,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60300 deadline: 1732710324158, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:24,163 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:24,163 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:24,163 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:24:24,167 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-27T12:24:24,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4. 2024-11-27T12:24:24,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4. as already flushing 2024-11-27T12:24:24,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4. 2024-11-27T12:24:24,167 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 953f259f29896493f17802c953dd7fd4, NAME => 'TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:24:24,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 953f259f29896493f17802c953dd7fd4, NAME => 'TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:24:24,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36215 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 953f259f29896493f17802c953dd7fd4, NAME => 'TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 953f259f29896493f17802c953dd7fd4, NAME => 'TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:24:24,171 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073741910_1086 (size=13085) 2024-11-27T12:24:24,186 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/.tmp/C/52ac39f85a304ccfa21c8d751673343d as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/C/52ac39f85a304ccfa21c8d751673343d 2024-11-27T12:24:24,203 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 953f259f29896493f17802c953dd7fd4/C of 953f259f29896493f17802c953dd7fd4 into 52ac39f85a304ccfa21c8d751673343d(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T12:24:24,203 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 953f259f29896493f17802c953dd7fd4: 2024-11-27T12:24:24,203 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., storeName=953f259f29896493f17802c953dd7fd4/C, priority=12, startTime=1732710263621; duration=0sec 2024-11-27T12:24:24,203 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T12:24:24,203 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 953f259f29896493f17802c953dd7fd4:C 2024-11-27T12:24:24,216 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=367 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/.tmp/B/2d7170a3422e4723a86e3315eb291250 2024-11-27T12:24:24,228 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/.tmp/C/739ec1f630f44e2da25c3f550cd56c59 is 50, key is test_row_0/C:col10/1732710263719/Put/seqid=0 2024-11-27T12:24:24,242 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073741911_1087 (size=12301) 2024-11-27T12:24:24,244 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=367 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/.tmp/C/739ec1f630f44e2da25c3f550cd56c59 2024-11-27T12:24:24,255 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/.tmp/A/36125348ec3f438fb20be16e467bbe30 as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/A/36125348ec3f438fb20be16e467bbe30 2024-11-27T12:24:24,266 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/A/36125348ec3f438fb20be16e467bbe30, entries=250, sequenceid=367, filesize=16.8 K 2024-11-27T12:24:24,267 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/.tmp/B/2d7170a3422e4723a86e3315eb291250 as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/B/2d7170a3422e4723a86e3315eb291250 2024-11-27T12:24:24,282 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/B/2d7170a3422e4723a86e3315eb291250, entries=150, sequenceid=367, filesize=12.0 K 2024-11-27T12:24:24,287 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/.tmp/C/739ec1f630f44e2da25c3f550cd56c59 as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/C/739ec1f630f44e2da25c3f550cd56c59 2024-11-27T12:24:24,295 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/C/739ec1f630f44e2da25c3f550cd56c59, entries=150, sequenceid=367, filesize=12.0 K 2024-11-27T12:24:24,296 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=114.05 KB/116790 for 953f259f29896493f17802c953dd7fd4 in 575ms, sequenceid=367, compaction requested=false 2024-11-27T12:24:24,296 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 953f259f29896493f17802c953dd7fd4: 2024-11-27T12:24:24,321 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-27T12:24:24,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4. 2024-11-27T12:24:24,321 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2902): Flushing 953f259f29896493f17802c953dd7fd4 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-27T12:24:24,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/.tmp/A/094673d7601d4b8f929833f6229cdcbb is 50, key is test_row_0/A:col10/1732710263757/Put/seqid=0 2024-11-27T12:24:24,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-27T12:24:24,336 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073741912_1088 (size=12301) 2024-11-27T12:24:24,337 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=390 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/.tmp/A/094673d7601d4b8f929833f6229cdcbb 2024-11-27T12:24:24,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/.tmp/B/f4fb913464a3424eb032d2743e5e7e79 is 50, key is test_row_0/B:col10/1732710263757/Put/seqid=0 2024-11-27T12:24:24,378 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073741913_1089 (size=12301) 2024-11-27T12:24:24,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(8855): Flush requested on 953f259f29896493f17802c953dd7fd4 2024-11-27T12:24:24,461 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4. as already flushing 2024-11-27T12:24:24,488 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:24,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60298 deadline: 1732710324487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:24,489 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:24,489 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:24,489 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:24:24,493 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:24,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60300 deadline: 1732710324488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:24,494 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:24,494 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:24,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60348 deadline: 1732710324489, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:24,494 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:24,494 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:24:24,495 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:24,495 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:24,495 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:24:24,602 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:24,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60298 deadline: 1732710324598, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:24,603 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:24,603 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:24,603 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:24,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60300 deadline: 1732710324598, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:24,603 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:24:24,603 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:24,603 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:24,603 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:24,603 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:24:24,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60348 deadline: 1732710324599, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:24,604 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:24,604 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:24,604 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:24:24,781 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=390 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/.tmp/B/f4fb913464a3424eb032d2743e5e7e79 2024-11-27T12:24:24,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/.tmp/C/9b76de2e5bda4e2ca2f2045ad3094694 is 50, key is test_row_0/C:col10/1732710263757/Put/seqid=0 2024-11-27T12:24:24,793 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073741914_1090 (size=12301) 2024-11-27T12:24:24,794 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=390 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/.tmp/C/9b76de2e5bda4e2ca2f2045ad3094694 2024-11-27T12:24:24,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/.tmp/A/094673d7601d4b8f929833f6229cdcbb as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/A/094673d7601d4b8f929833f6229cdcbb 2024-11-27T12:24:24,808 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/A/094673d7601d4b8f929833f6229cdcbb, entries=150, sequenceid=390, filesize=12.0 K 2024-11-27T12:24:24,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/.tmp/B/f4fb913464a3424eb032d2743e5e7e79 as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/B/f4fb913464a3424eb032d2743e5e7e79 2024-11-27T12:24:24,811 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:24,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60298 deadline: 1732710324808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:24,812 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:24,812 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:24,812 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:24,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60300 deadline: 1732710324808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:24,812 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:24:24,813 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:24,813 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:24,813 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:24,813 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:24:24,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60348 deadline: 1732710324809, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:24,814 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:24,815 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:24,815 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:24:24,818 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/B/f4fb913464a3424eb032d2743e5e7e79, entries=150, sequenceid=390, filesize=12.0 K 2024-11-27T12:24:24,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/.tmp/C/9b76de2e5bda4e2ca2f2045ad3094694 as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/C/9b76de2e5bda4e2ca2f2045ad3094694 2024-11-27T12:24:24,836 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/C/9b76de2e5bda4e2ca2f2045ad3094694, entries=150, sequenceid=390, filesize=12.0 K 2024-11-27T12:24:24,837 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=93.93 KB/96180 for 953f259f29896493f17802c953dd7fd4 in 516ms, sequenceid=390, compaction requested=true 2024-11-27T12:24:24,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2603): Flush status journal for 953f259f29896493f17802c953dd7fd4: 2024-11-27T12:24:24,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4. 2024-11-27T12:24:24,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=22 2024-11-27T12:24:24,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36215 {}] master.HMaster(4169): Remote procedure done, pid=22 2024-11-27T12:24:24,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-27T12:24:24,842 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=22, resume processing ppid=21 2024-11-27T12:24:24,843 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, ppid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1350 sec 2024-11-27T12:24:24,849 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees in 1.1460 sec 2024-11-27T12:24:25,330 INFO [AsyncFSWAL-0-hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c-prefix:1ba5af43fd99,36251,1732710241631 {}] wal.AbstractFSWAL(1368): Slow sync cost: 210 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:37513,DS-d207534c-4acd-4501-95b5-4ca3fb7f2487,DISK]] 2024-11-27T12:24:25,330 INFO [AsyncFSWAL-0-hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c-prefix:1ba5af43fd99,36251,1732710241631 {}] wal.AbstractFSWAL(1368): Slow sync cost: 208 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:37513,DS-d207534c-4acd-4501-95b5-4ca3fb7f2487,DISK]] 2024-11-27T12:24:25,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(8855): Flush requested on 953f259f29896493f17802c953dd7fd4 2024-11-27T12:24:25,331 INFO [AsyncFSWAL-0-hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c-prefix:1ba5af43fd99,36251,1732710241631 {}] wal.AbstractFSWAL(1368): Slow sync cost: 208 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:37513,DS-d207534c-4acd-4501-95b5-4ca3fb7f2487,DISK]] 2024-11-27T12:24:25,333 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T12:24:25,333 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 953f259f29896493f17802c953dd7fd4 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-27T12:24:25,342 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/.tmp/A/0ed4d815c41f4212b7d73bc0e824b4c9 is 50, key is test_row_0/A:col10/1732710265118/Put/seqid=0 2024-11-27T12:24:25,365 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:25,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60300 deadline: 1732710325362, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:25,366 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:25,366 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:25,366 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:25,366 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:24:25,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60348 deadline: 1732710325363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:25,366 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:25,366 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:25,366 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:24:25,369 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:25,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60298 deadline: 1732710325365, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:25,370 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:25,370 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:25,370 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:24:25,389 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073741915_1091 (size=17181) 2024-11-27T12:24:25,469 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:25,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60300 deadline: 1732710325468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:25,470 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:25,470 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:25,470 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:25,470 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:24:25,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60348 deadline: 1732710325468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:25,470 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:25,470 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:25,471 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:24:25,479 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:25,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60298 deadline: 1732710325477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:25,480 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:25,480 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:25,480 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:24:25,683 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:25,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60300 deadline: 1732710325682, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:25,683 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:25,683 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:25,684 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:25,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60348 deadline: 1732710325682, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:25,684 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:24:25,684 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:25,684 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:25,684 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:24:25,690 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:25,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60298 deadline: 1732710325688, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:25,690 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:25,690 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:25,690 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:24:25,789 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=410 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/.tmp/A/0ed4d815c41f4212b7d73bc0e824b4c9 2024-11-27T12:24:25,800 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/.tmp/B/7e5cfc853c6e48cc8de9b0f6ead58a30 is 50, key is test_row_0/B:col10/1732710265118/Put/seqid=0 2024-11-27T12:24:25,805 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073741916_1092 (size=12301) 2024-11-27T12:24:25,806 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=410 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/.tmp/B/7e5cfc853c6e48cc8de9b0f6ead58a30 2024-11-27T12:24:25,816 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/.tmp/C/27b7ab10f7a54c06a1ba61a97f292b8a is 50, key is test_row_0/C:col10/1732710265118/Put/seqid=0 2024-11-27T12:24:25,826 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073741917_1093 (size=12301) 2024-11-27T12:24:25,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-27T12:24:25,849 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T12:24:25,849 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36215 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-27T12:24:25,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36215 {}] procedure2.ProcedureExecutor(1139): Stored pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees 2024-11-27T12:24:25,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-27T12:24:25,851 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T12:24:25,852 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T12:24:25,852 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=24, ppid=23, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T12:24:25,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-27T12:24:25,989 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:25,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60300 deadline: 1732710325987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:25,989 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:25,989 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:25,989 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:25,989 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:24:25,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60348 deadline: 1732710325988, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:25,990 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:25,990 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:25,990 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:24:25,998 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:25,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60298 deadline: 1732710325998, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:25,999 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:25,999 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:25,999 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:24:26,005 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-27T12:24:26,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4. 2024-11-27T12:24:26,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4. as already flushing 2024-11-27T12:24:26,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4. 2024-11-27T12:24:26,005 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 953f259f29896493f17802c953dd7fd4, NAME => 'TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:24:26,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 953f259f29896493f17802c953dd7fd4, NAME => 'TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:24:26,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36215 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 953f259f29896493f17802c953dd7fd4, NAME => 'TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 953f259f29896493f17802c953dd7fd4, NAME => 'TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:24:26,158 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-27T12:24:26,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4. 2024-11-27T12:24:26,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4. as already flushing 2024-11-27T12:24:26,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4. 2024-11-27T12:24:26,158 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 953f259f29896493f17802c953dd7fd4, NAME => 'TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:24:26,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 953f259f29896493f17802c953dd7fd4, NAME => 'TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:24:26,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36215 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 953f259f29896493f17802c953dd7fd4, NAME => 'TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 953f259f29896493f17802c953dd7fd4, NAME => 'TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:24:26,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-27T12:24:26,227 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=410 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/.tmp/C/27b7ab10f7a54c06a1ba61a97f292b8a 2024-11-27T12:24:26,233 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/.tmp/A/0ed4d815c41f4212b7d73bc0e824b4c9 as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/A/0ed4d815c41f4212b7d73bc0e824b4c9 2024-11-27T12:24:26,238 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/A/0ed4d815c41f4212b7d73bc0e824b4c9, entries=250, sequenceid=410, filesize=16.8 K 2024-11-27T12:24:26,239 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/.tmp/B/7e5cfc853c6e48cc8de9b0f6ead58a30 as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/B/7e5cfc853c6e48cc8de9b0f6ead58a30 2024-11-27T12:24:26,246 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/B/7e5cfc853c6e48cc8de9b0f6ead58a30, entries=150, sequenceid=410, filesize=12.0 K 2024-11-27T12:24:26,248 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/.tmp/C/27b7ab10f7a54c06a1ba61a97f292b8a as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/C/27b7ab10f7a54c06a1ba61a97f292b8a 2024-11-27T12:24:26,254 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/C/27b7ab10f7a54c06a1ba61a97f292b8a, entries=150, sequenceid=410, filesize=12.0 K 2024-11-27T12:24:26,255 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=87.22 KB/89310 for 953f259f29896493f17802c953dd7fd4 in 922ms, sequenceid=410, compaction requested=true 2024-11-27T12:24:26,255 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 953f259f29896493f17802c953dd7fd4: 2024-11-27T12:24:26,255 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 953f259f29896493f17802c953dd7fd4:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T12:24:26,255 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T12:24:26,255 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-27T12:24:26,255 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 953f259f29896493f17802c953dd7fd4:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T12:24:26,255 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T12:24:26,255 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-27T12:24:26,255 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 953f259f29896493f17802c953dd7fd4:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T12:24:26,255 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T12:24:26,256 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 59748 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-27T12:24:26,256 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49988 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-27T12:24:26,256 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HStore(1541): 953f259f29896493f17802c953dd7fd4/A is initiating minor compaction (all files) 2024-11-27T12:24:26,256 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HStore(1541): 953f259f29896493f17802c953dd7fd4/B is initiating minor compaction (all files) 2024-11-27T12:24:26,257 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 953f259f29896493f17802c953dd7fd4/A in TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4. 2024-11-27T12:24:26,257 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 953f259f29896493f17802c953dd7fd4/B in TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4. 2024-11-27T12:24:26,257 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/B/1fe1487421454a07829081b7d1514263, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/B/2d7170a3422e4723a86e3315eb291250, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/B/f4fb913464a3424eb032d2743e5e7e79, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/B/7e5cfc853c6e48cc8de9b0f6ead58a30] into tmpdir=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/.tmp, totalSize=48.8 K 2024-11-27T12:24:26,257 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/A/11ab16b5c47847e29d71d7be88821602, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/A/36125348ec3f438fb20be16e467bbe30, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/A/094673d7601d4b8f929833f6229cdcbb, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/A/0ed4d815c41f4212b7d73bc0e824b4c9] into tmpdir=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/.tmp, totalSize=58.3 K 2024-11-27T12:24:26,258 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] compactions.Compactor(225): Compacting 11ab16b5c47847e29d71d7be88821602, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=351, earliestPutTs=1732710261883 2024-11-27T12:24:26,258 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.Compactor(225): Compacting 1fe1487421454a07829081b7d1514263, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=351, earliestPutTs=1732710261883 2024-11-27T12:24:26,258 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] compactions.Compactor(225): Compacting 36125348ec3f438fb20be16e467bbe30, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=367, earliestPutTs=1732710263074 2024-11-27T12:24:26,258 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.Compactor(225): Compacting 2d7170a3422e4723a86e3315eb291250, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=367, earliestPutTs=1732710263078 2024-11-27T12:24:26,258 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.Compactor(225): Compacting f4fb913464a3424eb032d2743e5e7e79, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=390, earliestPutTs=1732710263757 2024-11-27T12:24:26,258 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] compactions.Compactor(225): Compacting 094673d7601d4b8f929833f6229cdcbb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=390, earliestPutTs=1732710263757 2024-11-27T12:24:26,259 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.Compactor(225): Compacting 7e5cfc853c6e48cc8de9b0f6ead58a30, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=410, earliestPutTs=1732710265118 2024-11-27T12:24:26,259 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0ed4d815c41f4212b7d73bc0e824b4c9, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=410, earliestPutTs=1732710264474 2024-11-27T12:24:26,272 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 953f259f29896493f17802c953dd7fd4#A#compaction#81 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-27T12:24:26,273 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/.tmp/A/db85535426204d0d829fa4c2f33ede92 is 50, key is test_row_0/A:col10/1732710265118/Put/seqid=0 2024-11-27T12:24:26,273 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 953f259f29896493f17802c953dd7fd4#B#compaction#82 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T12:24:26,274 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/.tmp/B/d813107e6be040aab94aed6dc7c8807e is 50, key is test_row_0/B:col10/1732710265118/Put/seqid=0 2024-11-27T12:24:26,277 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073741918_1094 (size=13221) 2024-11-27T12:24:26,281 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073741919_1095 (size=13221) 2024-11-27T12:24:26,311 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-27T12:24:26,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4. 2024-11-27T12:24:26,311 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2902): Flushing 953f259f29896493f17802c953dd7fd4 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-27T12:24:26,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/.tmp/A/b55e8c306a4b4e17bf02caa7aa6b869b is 50, key is test_row_0/A:col10/1732710265362/Put/seqid=0 2024-11-27T12:24:26,320 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073741920_1096 (size=12301) 2024-11-27T12:24:26,321 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=426 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/.tmp/A/b55e8c306a4b4e17bf02caa7aa6b869b 2024-11-27T12:24:26,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/.tmp/B/eb05bca572354ecd9e4595b3f7ab1aa4 is 50, key is test_row_0/B:col10/1732710265362/Put/seqid=0 2024-11-27T12:24:26,336 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073741921_1097 (size=12301) 2024-11-27T12:24:26,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-27T12:24:26,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(8855): Flush requested on 953f259f29896493f17802c953dd7fd4 2024-11-27T12:24:26,502 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4. as already flushing 2024-11-27T12:24:26,532 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:26,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 181 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60348 deadline: 1732710326529, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:26,532 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:26,532 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:26,532 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:26,533 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:24:26,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 180 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60298 deadline: 1732710326530, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:26,533 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:26,533 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:26,533 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:24:26,539 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:26,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 180 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60300 deadline: 1732710326532, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:26,540 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:26,540 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:26,540 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:24:26,642 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:26,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 182 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60348 deadline: 1732710326638, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:26,643 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:26,643 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:26,643 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:26,643 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:24:26,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 181 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60298 deadline: 1732710326638, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:26,643 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:26,643 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:26,644 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:24:26,648 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:26,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 181 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60300 deadline: 1732710326647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:26,648 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:26,648 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:26,648 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:24:26,684 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/.tmp/A/db85535426204d0d829fa4c2f33ede92 as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/A/db85535426204d0d829fa4c2f33ede92 2024-11-27T12:24:26,690 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/.tmp/B/d813107e6be040aab94aed6dc7c8807e as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/B/d813107e6be040aab94aed6dc7c8807e 2024-11-27T12:24:26,690 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 953f259f29896493f17802c953dd7fd4/A of 953f259f29896493f17802c953dd7fd4 into db85535426204d0d829fa4c2f33ede92(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T12:24:26,691 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 953f259f29896493f17802c953dd7fd4: 2024-11-27T12:24:26,691 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., storeName=953f259f29896493f17802c953dd7fd4/A, priority=12, startTime=1732710266255; duration=0sec 2024-11-27T12:24:26,691 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T12:24:26,691 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 953f259f29896493f17802c953dd7fd4:A 2024-11-27T12:24:26,691 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-27T12:24:26,693 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49988 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-27T12:24:26,694 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HStore(1541): 953f259f29896493f17802c953dd7fd4/C is initiating minor compaction (all files) 2024-11-27T12:24:26,694 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 953f259f29896493f17802c953dd7fd4/C in TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4. 2024-11-27T12:24:26,694 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/C/52ac39f85a304ccfa21c8d751673343d, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/C/739ec1f630f44e2da25c3f550cd56c59, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/C/9b76de2e5bda4e2ca2f2045ad3094694, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/C/27b7ab10f7a54c06a1ba61a97f292b8a] into tmpdir=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/.tmp, totalSize=48.8 K 2024-11-27T12:24:26,694 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] compactions.Compactor(225): Compacting 52ac39f85a304ccfa21c8d751673343d, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=351, earliestPutTs=1732710261883 2024-11-27T12:24:26,695 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] compactions.Compactor(225): Compacting 739ec1f630f44e2da25c3f550cd56c59, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=367, earliestPutTs=1732710263078 2024-11-27T12:24:26,695 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9b76de2e5bda4e2ca2f2045ad3094694, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=390, earliestPutTs=1732710263757 2024-11-27T12:24:26,696 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] compactions.Compactor(225): Compacting 27b7ab10f7a54c06a1ba61a97f292b8a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=410, earliestPutTs=1732710265118 2024-11-27T12:24:26,697 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 953f259f29896493f17802c953dd7fd4/B of 953f259f29896493f17802c953dd7fd4 into d813107e6be040aab94aed6dc7c8807e(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T12:24:26,697 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 953f259f29896493f17802c953dd7fd4: 2024-11-27T12:24:26,697 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., storeName=953f259f29896493f17802c953dd7fd4/B, priority=12, startTime=1732710266255; duration=0sec 2024-11-27T12:24:26,698 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T12:24:26,698 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 953f259f29896493f17802c953dd7fd4:B 2024-11-27T12:24:26,707 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 953f259f29896493f17802c953dd7fd4#C#compaction#85 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T12:24:26,708 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/.tmp/C/134d20e9138b4770b61ac93c9611808b is 50, key is test_row_0/C:col10/1732710265118/Put/seqid=0 2024-11-27T12:24:26,715 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073741922_1098 (size=13221) 2024-11-27T12:24:26,737 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=426 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/.tmp/B/eb05bca572354ecd9e4595b3f7ab1aa4 2024-11-27T12:24:26,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/.tmp/C/9a127e4493a644d9b2de0d8eedb3a99f is 50, key is test_row_0/C:col10/1732710265362/Put/seqid=0 2024-11-27T12:24:26,753 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073741923_1099 (size=12301) 2024-11-27T12:24:26,754 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=426 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/.tmp/C/9a127e4493a644d9b2de0d8eedb3a99f 2024-11-27T12:24:26,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/.tmp/A/b55e8c306a4b4e17bf02caa7aa6b869b as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/A/b55e8c306a4b4e17bf02caa7aa6b869b 2024-11-27T12:24:26,769 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/A/b55e8c306a4b4e17bf02caa7aa6b869b, entries=150, sequenceid=426, filesize=12.0 K 2024-11-27T12:24:26,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/.tmp/B/eb05bca572354ecd9e4595b3f7ab1aa4 as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/B/eb05bca572354ecd9e4595b3f7ab1aa4 2024-11-27T12:24:26,776 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/B/eb05bca572354ecd9e4595b3f7ab1aa4, entries=150, sequenceid=426, filesize=12.0 K 2024-11-27T12:24:26,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/.tmp/C/9a127e4493a644d9b2de0d8eedb3a99f as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/C/9a127e4493a644d9b2de0d8eedb3a99f 2024-11-27T12:24:26,787 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/C/9a127e4493a644d9b2de0d8eedb3a99f, entries=150, sequenceid=426, filesize=12.0 K 2024-11-27T12:24:26,788 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=114.05 KB/116790 for 953f259f29896493f17802c953dd7fd4 in 477ms, sequenceid=426, compaction requested=false 2024-11-27T12:24:26,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2603): Flush status journal for 953f259f29896493f17802c953dd7fd4: 2024-11-27T12:24:26,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4. 2024-11-27T12:24:26,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=24 2024-11-27T12:24:26,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36215 {}] master.HMaster(4169): Remote procedure done, pid=24 2024-11-27T12:24:26,793 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=24, resume processing ppid=23 2024-11-27T12:24:26,793 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 939 msec 2024-11-27T12:24:26,795 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees in 944 msec 2024-11-27T12:24:26,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(8855): Flush requested on 953f259f29896493f17802c953dd7fd4 2024-11-27T12:24:26,851 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T12:24:26,851 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 953f259f29896493f17802c953dd7fd4 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-27T12:24:26,857 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/.tmp/A/65ab5a7a483747ab96b59a52270b5dd5 is 50, key is test_row_0/A:col10/1732710266527/Put/seqid=0 2024-11-27T12:24:26,863 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073741924_1100 (size=14741) 2024-11-27T12:24:26,864 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=449 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/.tmp/A/65ab5a7a483747ab96b59a52270b5dd5 2024-11-27T12:24:26,872 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:26,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 184 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60300 deadline: 1732710326871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:26,873 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:26,873 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:26,873 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:24:26,876 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:26,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 188 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60348 deadline: 1732710326872, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:26,876 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:26,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 188 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60298 deadline: 1732710326872, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:26,877 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:26,877 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:26,877 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:24:26,878 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/.tmp/B/fccfbb16619041aabb1b676ffa7b03cd is 50, key is test_row_0/B:col10/1732710266527/Put/seqid=0 2024-11-27T12:24:26,878 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:26,878 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:26,878 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:24:26,890 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073741925_1101 (size=12301) 2024-11-27T12:24:26,982 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:26,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 185 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60300 deadline: 1732710326978, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:26,982 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:26,982 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:26,982 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:24:26,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-27T12:24:26,989 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T12:24:26,989 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36215 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-27T12:24:26,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36215 {}] procedure2.ProcedureExecutor(1139): Stored pid=25, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees 2024-11-27T12:24:26,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-27T12:24:26,991 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=25, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T12:24:26,992 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=25, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T12:24:26,992 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=26, ppid=25, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T12:24:26,993 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:26,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 189 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60298 deadline: 1732710326989, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:26,993 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:26,993 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:26,993 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:26,993 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:24:26,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 189 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60348 deadline: 1732710326989, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:26,994 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:26,994 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:26,994 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:24:27,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-27T12:24:27,124 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/.tmp/C/134d20e9138b4770b61ac93c9611808b as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/C/134d20e9138b4770b61ac93c9611808b 2024-11-27T12:24:27,135 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 953f259f29896493f17802c953dd7fd4/C of 953f259f29896493f17802c953dd7fd4 into 134d20e9138b4770b61ac93c9611808b(size=12.9 K), total size for store is 24.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T12:24:27,135 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 953f259f29896493f17802c953dd7fd4: 2024-11-27T12:24:27,136 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., storeName=953f259f29896493f17802c953dd7fd4/C, priority=12, startTime=1732710266255; duration=0sec 2024-11-27T12:24:27,136 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T12:24:27,136 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 953f259f29896493f17802c953dd7fd4:C 2024-11-27T12:24:27,144 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-27T12:24:27,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4. 2024-11-27T12:24:27,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4. as already flushing 2024-11-27T12:24:27,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4. 2024-11-27T12:24:27,145 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => 953f259f29896493f17802c953dd7fd4, NAME => 'TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:24:27,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => 953f259f29896493f17802c953dd7fd4, NAME => 'TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:24:27,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36215 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 953f259f29896493f17802c953dd7fd4, NAME => 'TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 953f259f29896493f17802c953dd7fd4, NAME => 'TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:24:27,191 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:27,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 186 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60300 deadline: 1732710327188, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:27,192 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:27,192 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:27,192 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:24:27,201 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:27,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 190 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60348 deadline: 1732710327198, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:27,201 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:27,201 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:27,202 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:27,202 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:24:27,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 190 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60298 deadline: 1732710327198, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:27,202 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:27,202 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:27,202 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:24:27,291 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=449 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/.tmp/B/fccfbb16619041aabb1b676ffa7b03cd 2024-11-27T12:24:27,298 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-27T12:24:27,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4. 2024-11-27T12:24:27,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4. as already flushing 2024-11-27T12:24:27,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4. 2024-11-27T12:24:27,298 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => 953f259f29896493f17802c953dd7fd4, NAME => 'TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:24:27,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => 953f259f29896493f17802c953dd7fd4, NAME => 'TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:24:27,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36215 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 953f259f29896493f17802c953dd7fd4, NAME => 'TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 953f259f29896493f17802c953dd7fd4, NAME => 'TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:24:27,300 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/.tmp/C/db33fe3b2b6c41bd92a45d0dbb31df45 is 50, key is test_row_0/C:col10/1732710266527/Put/seqid=0 2024-11-27T12:24:27,304 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073741926_1102 (size=12301) 2024-11-27T12:24:27,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-27T12:24:27,451 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-27T12:24:27,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4. 2024-11-27T12:24:27,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4. as already flushing 2024-11-27T12:24:27,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4. 2024-11-27T12:24:27,451 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => 953f259f29896493f17802c953dd7fd4, NAME => 'TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:24:27,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => 953f259f29896493f17802c953dd7fd4, NAME => 'TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:24:27,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36215 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 953f259f29896493f17802c953dd7fd4, NAME => 'TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 953f259f29896493f17802c953dd7fd4, NAME => 'TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:24:27,501 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:27,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 187 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60300 deadline: 1732710327498, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:27,501 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:27,501 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:27,501 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:24:27,510 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:27,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 191 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60298 deadline: 1732710327508, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:27,511 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:27,511 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:27,511 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:27,511 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:24:27,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 191 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60348 deadline: 1732710327508, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:27,511 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:27,511 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:27,511 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:24:27,569 INFO [Thread-156 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-156. 2024-11-27T12:24:27,569 INFO [Thread-154 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-154. 2024-11-27T12:24:27,570 DEBUG [Thread-154 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T12:24:27,570 DEBUG [Thread-156 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T12:24:27,573 INFO [Thread-158 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-158. 2024-11-27T12:24:27,573 DEBUG [Thread-158 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T12:24:27,574 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T12:24:27,574 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T12:24:27,574 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T12:24:27,574 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T12:24:27,574 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T12:24:27,574 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T12:24:27,574 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T12:24:27,574 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T12:24:27,575 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T12:24:27,578 INFO [Thread-160 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-160. 2024-11-27T12:24:27,578 DEBUG [Thread-160 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T12:24:27,578 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T12:24:27,578 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T12:24:27,578 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T12:24:27,604 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-27T12:24:27,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4. 2024-11-27T12:24:27,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4. as already flushing 2024-11-27T12:24:27,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4. 2024-11-27T12:24:27,605 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => 953f259f29896493f17802c953dd7fd4, NAME => 'TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:24:27,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => 953f259f29896493f17802c953dd7fd4, NAME => 'TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:24:27,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36215 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 953f259f29896493f17802c953dd7fd4, NAME => 'TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 953f259f29896493f17802c953dd7fd4, NAME => 'TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:24:27,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-27T12:24:27,706 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=449 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/.tmp/C/db33fe3b2b6c41bd92a45d0dbb31df45 2024-11-27T12:24:27,717 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/.tmp/A/65ab5a7a483747ab96b59a52270b5dd5 as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/A/65ab5a7a483747ab96b59a52270b5dd5 2024-11-27T12:24:27,722 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/A/65ab5a7a483747ab96b59a52270b5dd5, entries=200, sequenceid=449, filesize=14.4 K 2024-11-27T12:24:27,723 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/.tmp/B/fccfbb16619041aabb1b676ffa7b03cd as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/B/fccfbb16619041aabb1b676ffa7b03cd 2024-11-27T12:24:27,728 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/B/fccfbb16619041aabb1b676ffa7b03cd, entries=150, sequenceid=449, filesize=12.0 K 2024-11-27T12:24:27,728 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:27,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60292 deadline: 1732710327728, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:27,729 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/.tmp/C/db33fe3b2b6c41bd92a45d0dbb31df45 as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/C/db33fe3b2b6c41bd92a45d0dbb31df45 2024-11-27T12:24:27,729 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 1ba5af43fd99,36251,1732710241631 for 'test_row_2' in 953f259f29896493f17802c953dd7fd4 of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18253 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T12:24:27,729 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:27,729 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:27,729 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:24:27,733 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/C/db33fe3b2b6c41bd92a45d0dbb31df45, entries=150, sequenceid=449, filesize=12.0 K 2024-11-27T12:24:27,734 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=80.51 KB/82440 for 953f259f29896493f17802c953dd7fd4 in 883ms, sequenceid=449, compaction requested=true 2024-11-27T12:24:27,734 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 953f259f29896493f17802c953dd7fd4: 2024-11-27T12:24:27,734 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 953f259f29896493f17802c953dd7fd4:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T12:24:27,734 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T12:24:27,734 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T12:24:27,735 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 953f259f29896493f17802c953dd7fd4:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T12:24:27,735 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T12:24:27,735 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T12:24:27,735 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 953f259f29896493f17802c953dd7fd4:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T12:24:27,735 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T12:24:27,736 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37823 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T12:24:27,736 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40263 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T12:24:27,736 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HStore(1541): 953f259f29896493f17802c953dd7fd4/B is initiating minor compaction (all files) 2024-11-27T12:24:27,736 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HStore(1541): 953f259f29896493f17802c953dd7fd4/A is initiating minor compaction (all files) 2024-11-27T12:24:27,736 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 953f259f29896493f17802c953dd7fd4/B in TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4. 2024-11-27T12:24:27,736 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 953f259f29896493f17802c953dd7fd4/A in TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4. 2024-11-27T12:24:27,736 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/B/d813107e6be040aab94aed6dc7c8807e, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/B/eb05bca572354ecd9e4595b3f7ab1aa4, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/B/fccfbb16619041aabb1b676ffa7b03cd] into tmpdir=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/.tmp, totalSize=36.9 K 2024-11-27T12:24:27,736 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/A/db85535426204d0d829fa4c2f33ede92, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/A/b55e8c306a4b4e17bf02caa7aa6b869b, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/A/65ab5a7a483747ab96b59a52270b5dd5] into tmpdir=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/.tmp, totalSize=39.3 K 2024-11-27T12:24:27,736 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.Compactor(225): Compacting d813107e6be040aab94aed6dc7c8807e, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=410, earliestPutTs=1732710265118 2024-11-27T12:24:27,736 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] compactions.Compactor(225): Compacting db85535426204d0d829fa4c2f33ede92, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=410, earliestPutTs=1732710265118 2024-11-27T12:24:27,736 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.Compactor(225): Compacting eb05bca572354ecd9e4595b3f7ab1aa4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=426, earliestPutTs=1732710265361 2024-11-27T12:24:27,737 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] compactions.Compactor(225): Compacting b55e8c306a4b4e17bf02caa7aa6b869b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=426, earliestPutTs=1732710265361 2024-11-27T12:24:27,737 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.Compactor(225): Compacting fccfbb16619041aabb1b676ffa7b03cd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=449, earliestPutTs=1732710266527 2024-11-27T12:24:27,737 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] compactions.Compactor(225): Compacting 65ab5a7a483747ab96b59a52270b5dd5, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=449, earliestPutTs=1732710266527 2024-11-27T12:24:27,744 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 953f259f29896493f17802c953dd7fd4#A#compaction#90 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T12:24:27,744 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 953f259f29896493f17802c953dd7fd4#B#compaction#91 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-27T12:24:27,744 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/.tmp/A/a1c1a891d7b649a0b884e8722fba2c03 is 50, key is test_row_0/A:col10/1732710266527/Put/seqid=0 2024-11-27T12:24:27,744 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/.tmp/B/71af1a70dd2448f3b0167fe2090a900d is 50, key is test_row_0/B:col10/1732710266527/Put/seqid=0 2024-11-27T12:24:27,749 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073741928_1104 (size=13323) 2024-11-27T12:24:27,749 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073741927_1103 (size=13323) 2024-11-27T12:24:27,758 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-27T12:24:27,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4. 2024-11-27T12:24:27,758 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2902): Flushing 953f259f29896493f17802c953dd7fd4 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-27T12:24:27,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/.tmp/A/54bc72d14895431e878429cb0d578741 is 50, key is test_row_0/A:col10/1732710266870/Put/seqid=0 2024-11-27T12:24:27,767 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073741929_1105 (size=12301) 2024-11-27T12:24:27,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(8855): Flush requested on 953f259f29896493f17802c953dd7fd4 2024-11-27T12:24:27,800 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4. as already flushing 2024-11-27T12:24:27,800 INFO [Thread-143 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-143. 2024-11-27T12:24:27,800 DEBUG [Thread-143 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T12:24:27,800 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T12:24:27,800 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T12:24:27,800 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T12:24:28,012 INFO [Thread-151 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-151. 2024-11-27T12:24:28,012 DEBUG [Thread-151 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T12:24:28,012 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T12:24:28,012 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T12:24:28,012 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T12:24:28,047 INFO [Thread-147 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-147. 2024-11-27T12:24:28,047 INFO [Thread-149 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-149. 2024-11-27T12:24:28,047 DEBUG [Thread-149 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T12:24:28,047 DEBUG [Thread-147 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T12:24:28,047 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T12:24:28,047 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T12:24:28,047 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T12:24:28,047 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T12:24:28,048 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T12:24:28,048 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T12:24:28,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-27T12:24:28,162 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/.tmp/A/a1c1a891d7b649a0b884e8722fba2c03 as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/A/a1c1a891d7b649a0b884e8722fba2c03 2024-11-27T12:24:28,162 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/.tmp/B/71af1a70dd2448f3b0167fe2090a900d as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/B/71af1a70dd2448f3b0167fe2090a900d 2024-11-27T12:24:28,167 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=465 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/.tmp/A/54bc72d14895431e878429cb0d578741 2024-11-27T12:24:28,169 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 953f259f29896493f17802c953dd7fd4/A of 953f259f29896493f17802c953dd7fd4 into a1c1a891d7b649a0b884e8722fba2c03(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T12:24:28,169 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 953f259f29896493f17802c953dd7fd4/B of 953f259f29896493f17802c953dd7fd4 into 71af1a70dd2448f3b0167fe2090a900d(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T12:24:28,169 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 953f259f29896493f17802c953dd7fd4: 2024-11-27T12:24:28,169 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 953f259f29896493f17802c953dd7fd4: 2024-11-27T12:24:28,169 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., storeName=953f259f29896493f17802c953dd7fd4/A, priority=13, startTime=1732710267734; duration=0sec 2024-11-27T12:24:28,169 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., storeName=953f259f29896493f17802c953dd7fd4/B, priority=13, startTime=1732710267735; duration=0sec 2024-11-27T12:24:28,169 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T12:24:28,169 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 953f259f29896493f17802c953dd7fd4:B 2024-11-27T12:24:28,169 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T12:24:28,169 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 953f259f29896493f17802c953dd7fd4:A 2024-11-27T12:24:28,169 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T12:24:28,170 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37823 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T12:24:28,170 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HStore(1541): 953f259f29896493f17802c953dd7fd4/C is initiating minor compaction (all files) 2024-11-27T12:24:28,170 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 953f259f29896493f17802c953dd7fd4/C in TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4. 2024-11-27T12:24:28,170 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/C/134d20e9138b4770b61ac93c9611808b, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/C/9a127e4493a644d9b2de0d8eedb3a99f, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/C/db33fe3b2b6c41bd92a45d0dbb31df45] into tmpdir=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/.tmp, totalSize=36.9 K 2024-11-27T12:24:28,171 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.Compactor(225): Compacting 134d20e9138b4770b61ac93c9611808b, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=410, earliestPutTs=1732710265118 2024-11-27T12:24:28,171 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.Compactor(225): Compacting 9a127e4493a644d9b2de0d8eedb3a99f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=426, earliestPutTs=1732710265361 2024-11-27T12:24:28,172 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.Compactor(225): Compacting db33fe3b2b6c41bd92a45d0dbb31df45, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=449, earliestPutTs=1732710266527 2024-11-27T12:24:28,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/.tmp/B/6e53f9d45212439db2e7455af6794c05 is 50, key is test_row_0/B:col10/1732710266870/Put/seqid=0 2024-11-27T12:24:28,179 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 953f259f29896493f17802c953dd7fd4#C#compaction#94 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T12:24:28,179 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073741930_1106 (size=12301) 2024-11-27T12:24:28,180 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/.tmp/C/61b84f6a65954df08a8e5c483cf3151c is 50, key is test_row_0/C:col10/1732710266527/Put/seqid=0 2024-11-27T12:24:28,184 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073741931_1107 (size=13323) 2024-11-27T12:24:28,581 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=465 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/.tmp/B/6e53f9d45212439db2e7455af6794c05 2024-11-27T12:24:28,594 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/.tmp/C/61b84f6a65954df08a8e5c483cf3151c as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/C/61b84f6a65954df08a8e5c483cf3151c 2024-11-27T12:24:28,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/.tmp/C/b754f52484ab4b298d7fc1ae0687d5a6 is 50, key is test_row_0/C:col10/1732710266870/Put/seqid=0 2024-11-27T12:24:28,602 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073741932_1108 (size=12301) 2024-11-27T12:24:28,602 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 953f259f29896493f17802c953dd7fd4/C of 953f259f29896493f17802c953dd7fd4 into 61b84f6a65954df08a8e5c483cf3151c(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T12:24:28,602 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 953f259f29896493f17802c953dd7fd4: 2024-11-27T12:24:28,602 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., storeName=953f259f29896493f17802c953dd7fd4/C, priority=13, startTime=1732710267735; duration=0sec 2024-11-27T12:24:28,603 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T12:24:28,603 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 953f259f29896493f17802c953dd7fd4:C 2024-11-27T12:24:29,003 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=465 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/.tmp/C/b754f52484ab4b298d7fc1ae0687d5a6 2024-11-27T12:24:29,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/.tmp/A/54bc72d14895431e878429cb0d578741 as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/A/54bc72d14895431e878429cb0d578741 2024-11-27T12:24:29,014 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/A/54bc72d14895431e878429cb0d578741, entries=150, sequenceid=465, filesize=12.0 K 2024-11-27T12:24:29,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/.tmp/B/6e53f9d45212439db2e7455af6794c05 as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/B/6e53f9d45212439db2e7455af6794c05 2024-11-27T12:24:29,019 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/B/6e53f9d45212439db2e7455af6794c05, entries=150, sequenceid=465, filesize=12.0 K 2024-11-27T12:24:29,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/.tmp/C/b754f52484ab4b298d7fc1ae0687d5a6 as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/C/b754f52484ab4b298d7fc1ae0687d5a6 2024-11-27T12:24:29,024 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/C/b754f52484ab4b298d7fc1ae0687d5a6, entries=150, sequenceid=465, filesize=12.0 K 2024-11-27T12:24:29,025 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=26.84 KB/27480 for 953f259f29896493f17802c953dd7fd4 in 1267ms, sequenceid=465, compaction requested=false 2024-11-27T12:24:29,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2603): Flush status journal for 953f259f29896493f17802c953dd7fd4: 2024-11-27T12:24:29,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4. 2024-11-27T12:24:29,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=26 2024-11-27T12:24:29,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36215 {}] master.HMaster(4169): Remote procedure done, pid=26 2024-11-27T12:24:29,029 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=26, resume processing ppid=25 2024-11-27T12:24:29,029 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=25, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0350 sec 2024-11-27T12:24:29,031 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees in 2.0400 sec 2024-11-27T12:24:29,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-27T12:24:29,139 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T12:24:30,014 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-27T12:24:37,779 INFO [Thread-145 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-145. 2024-11-27T12:24:37,780 DEBUG [Thread-145 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T12:24:37,780 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T12:24:37,780 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T12:24:37,780 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T12:24:37,780 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-27T12:24:37,780 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 16 2024-11-27T12:24:37,780 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 13 2024-11-27T12:24:37,780 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 117 2024-11-27T12:24:37,780 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 117 2024-11-27T12:24:37,780 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 113 2024-11-27T12:24:37,780 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-27T12:24:37,781 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2947 2024-11-27T12:24:37,781 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2888 2024-11-27T12:24:37,781 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-27T12:24:37,781 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1362 2024-11-27T12:24:37,781 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4085 rows 2024-11-27T12:24:37,781 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1364 2024-11-27T12:24:37,781 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4091 rows 2024-11-27T12:24:37,781 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-27T12:24:37,781 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMixedAtomicity(AcidGuaranteesTestBase.java:117) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-27T12:24:37,781 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T12:24:37,781 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T12:24:37,781 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T12:24:37,785 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36215 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-11-27T12:24:37,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36215 {}] procedure2.ProcedureExecutor(1139): Stored pid=27, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-27T12:24:37,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-27T12:24:37,792 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732710277791"}]},"ts":"1732710277791"} 2024-11-27T12:24:37,793 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-27T12:24:37,794 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-27T12:24:37,795 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=28, ppid=27, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-27T12:24:37,799 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=953f259f29896493f17802c953dd7fd4, UNASSIGN}] 2024-11-27T12:24:37,800 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=953f259f29896493f17802c953dd7fd4, UNASSIGN 2024-11-27T12:24:37,801 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=953f259f29896493f17802c953dd7fd4, regionState=CLOSING, regionLocation=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:37,803 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=953f259f29896493f17802c953dd7fd4, UNASSIGN because future has completed 2024-11-27T12:24:37,804 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-27T12:24:37,804 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=30, ppid=29, state=RUNNABLE, hasLock=false; CloseRegionProcedure 953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631}] 2024-11-27T12:24:37,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-27T12:24:37,965 INFO [RS_CLOSE_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] handler.UnassignRegionHandler(122): Close 953f259f29896493f17802c953dd7fd4 2024-11-27T12:24:37,965 DEBUG [RS_CLOSE_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-27T12:24:37,967 DEBUG [RS_CLOSE_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1722): Closing 953f259f29896493f17802c953dd7fd4, disabling compactions & flushes 2024-11-27T12:24:37,967 INFO [RS_CLOSE_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4. 2024-11-27T12:24:37,967 DEBUG [RS_CLOSE_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4. 2024-11-27T12:24:37,968 DEBUG [RS_CLOSE_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4. after waiting 0 ms 2024-11-27T12:24:37,968 DEBUG [RS_CLOSE_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4. 2024-11-27T12:24:37,968 INFO [RS_CLOSE_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(2902): Flushing 953f259f29896493f17802c953dd7fd4 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-11-27T12:24:37,975 DEBUG [RS_CLOSE_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/.tmp/A/4c8fc184ea4241f69df936d41192dee9 is 50, key is test_row_0/A:col10/1732710267798/Put/seqid=0 2024-11-27T12:24:37,980 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073741933_1109 (size=12301) 2024-11-27T12:24:38,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-27T12:24:38,382 INFO [RS_CLOSE_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=476 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/.tmp/A/4c8fc184ea4241f69df936d41192dee9 2024-11-27T12:24:38,397 DEBUG [RS_CLOSE_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/.tmp/B/1e8748bae58c435d8b3c2ecea29a4232 is 50, key is test_row_0/B:col10/1732710267798/Put/seqid=0 2024-11-27T12:24:38,401 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073741934_1110 (size=12301) 2024-11-27T12:24:38,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-27T12:24:38,804 INFO [RS_CLOSE_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=476 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/.tmp/B/1e8748bae58c435d8b3c2ecea29a4232 2024-11-27T12:24:38,812 DEBUG [RS_CLOSE_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/.tmp/C/cb3b0a9433764c09b3c82808a3908953 is 50, key is test_row_0/C:col10/1732710267798/Put/seqid=0 2024-11-27T12:24:38,815 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073741935_1111 (size=12301) 2024-11-27T12:24:38,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-27T12:24:39,217 INFO [RS_CLOSE_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=476 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/.tmp/C/cb3b0a9433764c09b3c82808a3908953 2024-11-27T12:24:39,229 DEBUG [RS_CLOSE_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/.tmp/A/4c8fc184ea4241f69df936d41192dee9 as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/A/4c8fc184ea4241f69df936d41192dee9 2024-11-27T12:24:39,234 INFO [RS_CLOSE_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/A/4c8fc184ea4241f69df936d41192dee9, entries=150, sequenceid=476, filesize=12.0 K 2024-11-27T12:24:39,235 DEBUG [RS_CLOSE_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/.tmp/B/1e8748bae58c435d8b3c2ecea29a4232 as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/B/1e8748bae58c435d8b3c2ecea29a4232 2024-11-27T12:24:39,240 INFO [RS_CLOSE_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/B/1e8748bae58c435d8b3c2ecea29a4232, entries=150, sequenceid=476, filesize=12.0 K 2024-11-27T12:24:39,241 DEBUG [RS_CLOSE_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/.tmp/C/cb3b0a9433764c09b3c82808a3908953 as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/C/cb3b0a9433764c09b3c82808a3908953 2024-11-27T12:24:39,246 INFO [RS_CLOSE_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/C/cb3b0a9433764c09b3c82808a3908953, entries=150, sequenceid=476, filesize=12.0 K 2024-11-27T12:24:39,247 INFO [RS_CLOSE_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for 953f259f29896493f17802c953dd7fd4 in 1279ms, sequenceid=476, compaction requested=true 2024-11-27T12:24:39,248 DEBUG [StoreCloser-TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/A/8099951072fb4accbd57bdc559a2bf3d, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/A/64eaaaf2596d48f48d94dde59111acdc, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/A/e3c3fedbfa7147cfb5a23ccf66090521, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/A/c77e507f29524cd1b12ee7dfed66f019, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/A/a6c580332c9042ce80de73f6a68a90f5, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/A/68e4df30616f480c975bf82bc491a7bc, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/A/e56e87dcf7ce477f9248e78c39bfd4ac, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/A/6a05cb9776084131a562fd3e67e8f3cb, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/A/aca784817a654817a555b448746a7ae8, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/A/14c423e1e7914d0f8488ef8648517e1a, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/A/4cea0403883b4e6090566caa7b04f5c4, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/A/63bd5a4637144099bf2ebad11527a6df, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/A/d0f468dde04040e0aef7f77be7637e5c, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/A/26dbdbed0055463dbd2b17335bd10d22, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/A/47295f031e6c4a7cbbf1ce4923cabb48, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/A/0290e3cdd8874318b8965ad3e503f629, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/A/27c79de79e7b4394b881b28869dab96f, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/A/0c5b478aa7a94bf2a41531c200cb964a, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/A/872c35d628a040e2afa1f088e7118972, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/A/316de21f553f4d0a95a28b20b817f880, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/A/e46a93be90de468bb732984bda76f025, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/A/5c326aa08229460cae619c871d8137ff, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/A/4a0b78d52e164759b66147d4beae2173, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/A/11ab16b5c47847e29d71d7be88821602, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/A/36125348ec3f438fb20be16e467bbe30, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/A/094673d7601d4b8f929833f6229cdcbb, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/A/0ed4d815c41f4212b7d73bc0e824b4c9, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/A/db85535426204d0d829fa4c2f33ede92, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/A/b55e8c306a4b4e17bf02caa7aa6b869b, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/A/65ab5a7a483747ab96b59a52270b5dd5] to archive 2024-11-27T12:24:39,251 DEBUG [StoreCloser-TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-27T12:24:39,257 DEBUG [StoreCloser-TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/A/8099951072fb4accbd57bdc559a2bf3d to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/A/8099951072fb4accbd57bdc559a2bf3d 2024-11-27T12:24:39,259 DEBUG [StoreCloser-TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/A/64eaaaf2596d48f48d94dde59111acdc to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/A/64eaaaf2596d48f48d94dde59111acdc 2024-11-27T12:24:39,260 DEBUG [StoreCloser-TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/A/e3c3fedbfa7147cfb5a23ccf66090521 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/A/e3c3fedbfa7147cfb5a23ccf66090521 2024-11-27T12:24:39,262 DEBUG [StoreCloser-TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/A/c77e507f29524cd1b12ee7dfed66f019 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/A/c77e507f29524cd1b12ee7dfed66f019 2024-11-27T12:24:39,263 DEBUG [StoreCloser-TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/A/a6c580332c9042ce80de73f6a68a90f5 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/A/a6c580332c9042ce80de73f6a68a90f5 2024-11-27T12:24:39,265 DEBUG [StoreCloser-TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/A/68e4df30616f480c975bf82bc491a7bc to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/A/68e4df30616f480c975bf82bc491a7bc 2024-11-27T12:24:39,267 DEBUG [StoreCloser-TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/A/e56e87dcf7ce477f9248e78c39bfd4ac to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/A/e56e87dcf7ce477f9248e78c39bfd4ac 2024-11-27T12:24:39,268 DEBUG [StoreCloser-TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/A/6a05cb9776084131a562fd3e67e8f3cb to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/A/6a05cb9776084131a562fd3e67e8f3cb 2024-11-27T12:24:39,270 DEBUG [StoreCloser-TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/A/aca784817a654817a555b448746a7ae8 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/A/aca784817a654817a555b448746a7ae8 2024-11-27T12:24:39,271 DEBUG [StoreCloser-TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/A/14c423e1e7914d0f8488ef8648517e1a to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/A/14c423e1e7914d0f8488ef8648517e1a 2024-11-27T12:24:39,273 DEBUG [StoreCloser-TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/A/4cea0403883b4e6090566caa7b04f5c4 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/A/4cea0403883b4e6090566caa7b04f5c4 2024-11-27T12:24:39,274 DEBUG [StoreCloser-TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/A/63bd5a4637144099bf2ebad11527a6df to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/A/63bd5a4637144099bf2ebad11527a6df 2024-11-27T12:24:39,276 DEBUG [StoreCloser-TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/A/d0f468dde04040e0aef7f77be7637e5c to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/A/d0f468dde04040e0aef7f77be7637e5c 2024-11-27T12:24:39,277 DEBUG [StoreCloser-TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/A/26dbdbed0055463dbd2b17335bd10d22 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/A/26dbdbed0055463dbd2b17335bd10d22 2024-11-27T12:24:39,278 DEBUG [StoreCloser-TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/A/47295f031e6c4a7cbbf1ce4923cabb48 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/A/47295f031e6c4a7cbbf1ce4923cabb48 2024-11-27T12:24:39,279 DEBUG [StoreCloser-TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/A/0290e3cdd8874318b8965ad3e503f629 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/A/0290e3cdd8874318b8965ad3e503f629 2024-11-27T12:24:39,281 DEBUG [StoreCloser-TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/A/27c79de79e7b4394b881b28869dab96f to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/A/27c79de79e7b4394b881b28869dab96f 2024-11-27T12:24:39,282 DEBUG [StoreCloser-TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/A/0c5b478aa7a94bf2a41531c200cb964a to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/A/0c5b478aa7a94bf2a41531c200cb964a 2024-11-27T12:24:39,283 DEBUG [StoreCloser-TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/A/872c35d628a040e2afa1f088e7118972 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/A/872c35d628a040e2afa1f088e7118972 2024-11-27T12:24:39,284 DEBUG [StoreCloser-TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/A/316de21f553f4d0a95a28b20b817f880 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/A/316de21f553f4d0a95a28b20b817f880 2024-11-27T12:24:39,286 DEBUG [StoreCloser-TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/A/e46a93be90de468bb732984bda76f025 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/A/e46a93be90de468bb732984bda76f025 2024-11-27T12:24:39,287 DEBUG [StoreCloser-TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/A/5c326aa08229460cae619c871d8137ff to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/A/5c326aa08229460cae619c871d8137ff 2024-11-27T12:24:39,288 DEBUG [StoreCloser-TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/A/4a0b78d52e164759b66147d4beae2173 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/A/4a0b78d52e164759b66147d4beae2173 2024-11-27T12:24:39,289 DEBUG [StoreCloser-TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/A/11ab16b5c47847e29d71d7be88821602 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/A/11ab16b5c47847e29d71d7be88821602 2024-11-27T12:24:39,290 DEBUG [StoreCloser-TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/A/36125348ec3f438fb20be16e467bbe30 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/A/36125348ec3f438fb20be16e467bbe30 2024-11-27T12:24:39,291 DEBUG [StoreCloser-TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/A/094673d7601d4b8f929833f6229cdcbb to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/A/094673d7601d4b8f929833f6229cdcbb 2024-11-27T12:24:39,292 DEBUG [StoreCloser-TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/A/0ed4d815c41f4212b7d73bc0e824b4c9 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/A/0ed4d815c41f4212b7d73bc0e824b4c9 2024-11-27T12:24:39,293 DEBUG [StoreCloser-TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/A/db85535426204d0d829fa4c2f33ede92 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/A/db85535426204d0d829fa4c2f33ede92 2024-11-27T12:24:39,294 DEBUG [StoreCloser-TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/A/b55e8c306a4b4e17bf02caa7aa6b869b to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/A/b55e8c306a4b4e17bf02caa7aa6b869b 2024-11-27T12:24:39,295 DEBUG [StoreCloser-TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/A/65ab5a7a483747ab96b59a52270b5dd5 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/A/65ab5a7a483747ab96b59a52270b5dd5 2024-11-27T12:24:39,308 DEBUG [StoreCloser-TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/B/73374052d1204864af13309c8b4250ed, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/B/a6c5abf3e4774096bf5225521e4820dc, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/B/0e7d5f462461482e970041c4a1fb2dac, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/B/a0f5d8167eeb4429b13790a4e760f1bd, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/B/c146c96db80c4b2985d7601d28386162, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/B/048712a3d3244a8799d60dc5fd1312e8, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/B/2a35b97e787c4035a00a16b41757adc4, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/B/4356547e1d5f47b3b1a12539cc3a6a3e, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/B/62133eb7cc03471aa362cef0dbd6e620, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/B/ddc5a2d40cab425f81fc2065b32ee7f2, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/B/25d3820f844e41a8ba05baf59f997efc, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/B/7474690934c34fc8a49810280d442f89, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/B/49a8dc03ebad4782a37eafc183a6ead7, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/B/434461a16faa4cf88fa32a5682bb25dd, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/B/c886c2a649564c6fbbc97f98416226d3, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/B/0c94518a184f4bd3ab886ee201ec40ff, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/B/112397a2d108400388bb19842248befb, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/B/8313896b4eff469fbf570cdc121e3c58, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/B/5a9bb0d6c9654e21ac382e3ce031e732, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/B/198c382a6c1b428bbbedd5c3c1fc3ba1, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/B/3393a10bd3e84fd09c57b1570dc60cc1, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/B/e4096f31d0df4eac93c447ad55abc18f, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/B/1fe1487421454a07829081b7d1514263, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/B/fc4015eab12549d7a02f15cf829b47c8, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/B/2d7170a3422e4723a86e3315eb291250, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/B/f4fb913464a3424eb032d2743e5e7e79, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/B/d813107e6be040aab94aed6dc7c8807e, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/B/7e5cfc853c6e48cc8de9b0f6ead58a30, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/B/eb05bca572354ecd9e4595b3f7ab1aa4, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/B/fccfbb16619041aabb1b676ffa7b03cd] to archive 2024-11-27T12:24:39,309 DEBUG [StoreCloser-TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-27T12:24:39,311 DEBUG [StoreCloser-TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/B/73374052d1204864af13309c8b4250ed to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/B/73374052d1204864af13309c8b4250ed 2024-11-27T12:24:39,313 DEBUG [StoreCloser-TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/B/a6c5abf3e4774096bf5225521e4820dc to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/B/a6c5abf3e4774096bf5225521e4820dc 2024-11-27T12:24:39,314 DEBUG [StoreCloser-TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/B/0e7d5f462461482e970041c4a1fb2dac to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/B/0e7d5f462461482e970041c4a1fb2dac 2024-11-27T12:24:39,315 DEBUG [StoreCloser-TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/B/a0f5d8167eeb4429b13790a4e760f1bd to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/B/a0f5d8167eeb4429b13790a4e760f1bd 2024-11-27T12:24:39,317 DEBUG [StoreCloser-TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/B/c146c96db80c4b2985d7601d28386162 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/B/c146c96db80c4b2985d7601d28386162 2024-11-27T12:24:39,318 DEBUG [StoreCloser-TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/B/048712a3d3244a8799d60dc5fd1312e8 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/B/048712a3d3244a8799d60dc5fd1312e8 2024-11-27T12:24:39,319 DEBUG [StoreCloser-TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/B/2a35b97e787c4035a00a16b41757adc4 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/B/2a35b97e787c4035a00a16b41757adc4 2024-11-27T12:24:39,320 DEBUG [StoreCloser-TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/B/4356547e1d5f47b3b1a12539cc3a6a3e to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/B/4356547e1d5f47b3b1a12539cc3a6a3e 2024-11-27T12:24:39,322 DEBUG [StoreCloser-TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/B/62133eb7cc03471aa362cef0dbd6e620 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/B/62133eb7cc03471aa362cef0dbd6e620 2024-11-27T12:24:39,323 DEBUG [StoreCloser-TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/B/ddc5a2d40cab425f81fc2065b32ee7f2 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/B/ddc5a2d40cab425f81fc2065b32ee7f2 2024-11-27T12:24:39,324 DEBUG [StoreCloser-TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/B/25d3820f844e41a8ba05baf59f997efc to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/B/25d3820f844e41a8ba05baf59f997efc 2024-11-27T12:24:39,325 DEBUG [StoreCloser-TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/B/7474690934c34fc8a49810280d442f89 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/B/7474690934c34fc8a49810280d442f89 2024-11-27T12:24:39,326 DEBUG [StoreCloser-TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/B/49a8dc03ebad4782a37eafc183a6ead7 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/B/49a8dc03ebad4782a37eafc183a6ead7 2024-11-27T12:24:39,327 DEBUG [StoreCloser-TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/B/434461a16faa4cf88fa32a5682bb25dd to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/B/434461a16faa4cf88fa32a5682bb25dd 2024-11-27T12:24:39,328 DEBUG [StoreCloser-TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/B/c886c2a649564c6fbbc97f98416226d3 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/B/c886c2a649564c6fbbc97f98416226d3 2024-11-27T12:24:39,329 DEBUG [StoreCloser-TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/B/0c94518a184f4bd3ab886ee201ec40ff to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/B/0c94518a184f4bd3ab886ee201ec40ff 2024-11-27T12:24:39,330 DEBUG [StoreCloser-TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/B/112397a2d108400388bb19842248befb to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/B/112397a2d108400388bb19842248befb 2024-11-27T12:24:39,331 DEBUG [StoreCloser-TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/B/8313896b4eff469fbf570cdc121e3c58 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/B/8313896b4eff469fbf570cdc121e3c58 2024-11-27T12:24:39,332 DEBUG [StoreCloser-TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/B/5a9bb0d6c9654e21ac382e3ce031e732 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/B/5a9bb0d6c9654e21ac382e3ce031e732 2024-11-27T12:24:39,333 DEBUG [StoreCloser-TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/B/198c382a6c1b428bbbedd5c3c1fc3ba1 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/B/198c382a6c1b428bbbedd5c3c1fc3ba1 2024-11-27T12:24:39,334 DEBUG [StoreCloser-TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/B/3393a10bd3e84fd09c57b1570dc60cc1 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/B/3393a10bd3e84fd09c57b1570dc60cc1 2024-11-27T12:24:39,335 DEBUG [StoreCloser-TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/B/e4096f31d0df4eac93c447ad55abc18f to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/B/e4096f31d0df4eac93c447ad55abc18f 2024-11-27T12:24:39,336 DEBUG [StoreCloser-TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/B/1fe1487421454a07829081b7d1514263 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/B/1fe1487421454a07829081b7d1514263 2024-11-27T12:24:39,337 DEBUG [StoreCloser-TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/B/fc4015eab12549d7a02f15cf829b47c8 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/B/fc4015eab12549d7a02f15cf829b47c8 2024-11-27T12:24:39,338 DEBUG [StoreCloser-TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/B/2d7170a3422e4723a86e3315eb291250 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/B/2d7170a3422e4723a86e3315eb291250 2024-11-27T12:24:39,339 DEBUG [StoreCloser-TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/B/f4fb913464a3424eb032d2743e5e7e79 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/B/f4fb913464a3424eb032d2743e5e7e79 2024-11-27T12:24:39,340 DEBUG [StoreCloser-TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/B/d813107e6be040aab94aed6dc7c8807e to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/B/d813107e6be040aab94aed6dc7c8807e 2024-11-27T12:24:39,341 DEBUG [StoreCloser-TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/B/7e5cfc853c6e48cc8de9b0f6ead58a30 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/B/7e5cfc853c6e48cc8de9b0f6ead58a30 2024-11-27T12:24:39,342 DEBUG [StoreCloser-TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/B/eb05bca572354ecd9e4595b3f7ab1aa4 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/B/eb05bca572354ecd9e4595b3f7ab1aa4 2024-11-27T12:24:39,343 DEBUG [StoreCloser-TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/B/fccfbb16619041aabb1b676ffa7b03cd to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/B/fccfbb16619041aabb1b676ffa7b03cd 2024-11-27T12:24:39,345 DEBUG [StoreCloser-TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/C/9994adf73fb04d6b95011b5baf512bfb, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/C/1d99c02af3404b978d18793ad3e3a6dc, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/C/ca75ebc244404644b7ddbdb9b875f012, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/C/e439ae32af0b4328bf72d5106a7e9cbe, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/C/551ca4808fab40b493220a40b15312bd, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/C/e0864f52a7ca4630b2b957d17bded558, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/C/d047d654ce40415780ab3e93779afec5, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/C/3eca934aa97f43cb982022ae17077462, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/C/20ba43d95b3449329b7c56e9754ba8c9, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/C/826694515f524af08e9fa77d290e62d7, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/C/a272e3ce38ae4891b7b55842fc31aed0, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/C/f9915d19089f425eaeaeb975bd4396ed, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/C/b9112910c84d4698b52b5c5ba3428703, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/C/93eb828966694f95a04383d56b082129, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/C/25dc0f5080c6444c9f9e749c47c5cc7b, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/C/041f7410542a4874816bf4175bebd6fb, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/C/79f9d27b63cb480cadb48c3e55825b7d, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/C/23b9ea599d97432684675b1826a49ee1, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/C/eedf2d8af6904ff6a7700d253761aaa5, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/C/d04fdd9a5c054303988cc48465fdd43a, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/C/853981e7bd1c42d28479f3287a5718d4, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/C/6950ed486d28459c8a25da07153e51a3, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/C/52ac39f85a304ccfa21c8d751673343d, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/C/5a4c44a3b52f4c21a8fb34bd6de79ace, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/C/739ec1f630f44e2da25c3f550cd56c59, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/C/9b76de2e5bda4e2ca2f2045ad3094694, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/C/134d20e9138b4770b61ac93c9611808b, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/C/27b7ab10f7a54c06a1ba61a97f292b8a, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/C/9a127e4493a644d9b2de0d8eedb3a99f, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/C/db33fe3b2b6c41bd92a45d0dbb31df45] to archive 2024-11-27T12:24:39,346 DEBUG [StoreCloser-TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-27T12:24:39,348 DEBUG [StoreCloser-TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/C/9994adf73fb04d6b95011b5baf512bfb to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/C/9994adf73fb04d6b95011b5baf512bfb 2024-11-27T12:24:39,349 DEBUG [StoreCloser-TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/C/1d99c02af3404b978d18793ad3e3a6dc to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/C/1d99c02af3404b978d18793ad3e3a6dc 2024-11-27T12:24:39,350 DEBUG [StoreCloser-TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/C/ca75ebc244404644b7ddbdb9b875f012 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/C/ca75ebc244404644b7ddbdb9b875f012 2024-11-27T12:24:39,351 DEBUG [StoreCloser-TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/C/e439ae32af0b4328bf72d5106a7e9cbe to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/C/e439ae32af0b4328bf72d5106a7e9cbe 2024-11-27T12:24:39,352 DEBUG [StoreCloser-TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/C/551ca4808fab40b493220a40b15312bd to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/C/551ca4808fab40b493220a40b15312bd 2024-11-27T12:24:39,353 DEBUG [StoreCloser-TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/C/e0864f52a7ca4630b2b957d17bded558 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/C/e0864f52a7ca4630b2b957d17bded558 2024-11-27T12:24:39,354 DEBUG [StoreCloser-TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/C/d047d654ce40415780ab3e93779afec5 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/C/d047d654ce40415780ab3e93779afec5 2024-11-27T12:24:39,355 DEBUG [StoreCloser-TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/C/3eca934aa97f43cb982022ae17077462 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/C/3eca934aa97f43cb982022ae17077462 2024-11-27T12:24:39,356 DEBUG [StoreCloser-TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/C/20ba43d95b3449329b7c56e9754ba8c9 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/C/20ba43d95b3449329b7c56e9754ba8c9 2024-11-27T12:24:39,356 DEBUG [StoreCloser-TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/C/826694515f524af08e9fa77d290e62d7 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/C/826694515f524af08e9fa77d290e62d7 2024-11-27T12:24:39,358 DEBUG [StoreCloser-TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/C/a272e3ce38ae4891b7b55842fc31aed0 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/C/a272e3ce38ae4891b7b55842fc31aed0 2024-11-27T12:24:39,358 DEBUG [StoreCloser-TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/C/f9915d19089f425eaeaeb975bd4396ed to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/C/f9915d19089f425eaeaeb975bd4396ed 2024-11-27T12:24:39,359 DEBUG [StoreCloser-TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/C/b9112910c84d4698b52b5c5ba3428703 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/C/b9112910c84d4698b52b5c5ba3428703 2024-11-27T12:24:39,360 DEBUG [StoreCloser-TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/C/93eb828966694f95a04383d56b082129 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/C/93eb828966694f95a04383d56b082129 2024-11-27T12:24:39,361 DEBUG [StoreCloser-TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/C/25dc0f5080c6444c9f9e749c47c5cc7b to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/C/25dc0f5080c6444c9f9e749c47c5cc7b 2024-11-27T12:24:39,362 DEBUG [StoreCloser-TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/C/041f7410542a4874816bf4175bebd6fb to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/C/041f7410542a4874816bf4175bebd6fb 2024-11-27T12:24:39,364 DEBUG [StoreCloser-TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/C/79f9d27b63cb480cadb48c3e55825b7d to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/C/79f9d27b63cb480cadb48c3e55825b7d 2024-11-27T12:24:39,365 DEBUG [StoreCloser-TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/C/23b9ea599d97432684675b1826a49ee1 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/C/23b9ea599d97432684675b1826a49ee1 2024-11-27T12:24:39,366 DEBUG [StoreCloser-TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/C/eedf2d8af6904ff6a7700d253761aaa5 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/C/eedf2d8af6904ff6a7700d253761aaa5 2024-11-27T12:24:39,367 DEBUG [StoreCloser-TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/C/d04fdd9a5c054303988cc48465fdd43a to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/C/d04fdd9a5c054303988cc48465fdd43a 2024-11-27T12:24:39,368 DEBUG [StoreCloser-TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/C/853981e7bd1c42d28479f3287a5718d4 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/C/853981e7bd1c42d28479f3287a5718d4 2024-11-27T12:24:39,369 DEBUG [StoreCloser-TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/C/6950ed486d28459c8a25da07153e51a3 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/C/6950ed486d28459c8a25da07153e51a3 2024-11-27T12:24:39,370 DEBUG [StoreCloser-TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/C/52ac39f85a304ccfa21c8d751673343d to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/C/52ac39f85a304ccfa21c8d751673343d 2024-11-27T12:24:39,371 DEBUG [StoreCloser-TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/C/5a4c44a3b52f4c21a8fb34bd6de79ace to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/C/5a4c44a3b52f4c21a8fb34bd6de79ace 2024-11-27T12:24:39,373 DEBUG [StoreCloser-TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/C/739ec1f630f44e2da25c3f550cd56c59 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/C/739ec1f630f44e2da25c3f550cd56c59 2024-11-27T12:24:39,374 DEBUG [StoreCloser-TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/C/9b76de2e5bda4e2ca2f2045ad3094694 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/C/9b76de2e5bda4e2ca2f2045ad3094694 2024-11-27T12:24:39,375 DEBUG [StoreCloser-TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/C/134d20e9138b4770b61ac93c9611808b to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/C/134d20e9138b4770b61ac93c9611808b 2024-11-27T12:24:39,377 DEBUG [StoreCloser-TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/C/27b7ab10f7a54c06a1ba61a97f292b8a to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/C/27b7ab10f7a54c06a1ba61a97f292b8a 2024-11-27T12:24:39,378 DEBUG [StoreCloser-TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/C/9a127e4493a644d9b2de0d8eedb3a99f to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/C/9a127e4493a644d9b2de0d8eedb3a99f 2024-11-27T12:24:39,380 DEBUG [StoreCloser-TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/C/db33fe3b2b6c41bd92a45d0dbb31df45 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/C/db33fe3b2b6c41bd92a45d0dbb31df45 2024-11-27T12:24:39,386 DEBUG [RS_CLOSE_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/recovered.edits/479.seqid, newMaxSeqId=479, maxSeqId=1 2024-11-27T12:24:39,387 INFO [RS_CLOSE_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4. 2024-11-27T12:24:39,387 DEBUG [RS_CLOSE_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1676): Region close journal for 953f259f29896493f17802c953dd7fd4: Waiting for close lock at 1732710277967Running coprocessor pre-close hooks at 1732710277967Disabling compacts and flushes for region at 1732710277967Disabling writes for close at 1732710277968 (+1 ms)Obtaining lock to block concurrent updates at 1732710277968Preparing flush snapshotting stores in 953f259f29896493f17802c953dd7fd4 at 1732710277968Finished memstore snapshotting TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=90720, getOffHeapSize=0, getCellsCount=750 at 1732710277969 (+1 ms)Flushing stores of TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4. at 1732710277969Flushing 953f259f29896493f17802c953dd7fd4/A: creating writer at 1732710277969Flushing 953f259f29896493f17802c953dd7fd4/A: appending metadata at 1732710277974 (+5 ms)Flushing 953f259f29896493f17802c953dd7fd4/A: closing flushed file at 1732710277974Flushing 953f259f29896493f17802c953dd7fd4/B: creating writer at 1732710278393 (+419 ms)Flushing 953f259f29896493f17802c953dd7fd4/B: appending metadata at 1732710278397 (+4 ms)Flushing 953f259f29896493f17802c953dd7fd4/B: closing flushed file at 1732710278397Flushing 953f259f29896493f17802c953dd7fd4/C: creating writer at 1732710278809 (+412 ms)Flushing 953f259f29896493f17802c953dd7fd4/C: appending metadata at 1732710278811 (+2 ms)Flushing 953f259f29896493f17802c953dd7fd4/C: closing flushed file at 1732710278811Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5d65b9f3: reopening flushed file at 1732710279228 (+417 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@746c4672: reopening flushed file at 1732710279234 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4a4d943f: reopening flushed file at 1732710279240 (+6 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for 953f259f29896493f17802c953dd7fd4 in 1279ms, sequenceid=476, compaction requested=true at 1732710279247 (+7 ms)Writing region close event to WAL at 1732710279382 (+135 ms)Running coprocessor post-close hooks at 1732710279387 (+5 ms)Closed at 1732710279387 2024-11-27T12:24:39,390 INFO [RS_CLOSE_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] handler.UnassignRegionHandler(157): Closed 953f259f29896493f17802c953dd7fd4 2024-11-27T12:24:39,391 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=953f259f29896493f17802c953dd7fd4, regionState=CLOSED 2024-11-27T12:24:39,393 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=30, ppid=29, state=RUNNABLE, hasLock=false; CloseRegionProcedure 953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 because future has completed 2024-11-27T12:24:39,397 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=30, resume processing ppid=29 2024-11-27T12:24:39,397 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, ppid=29, state=SUCCESS, hasLock=false; CloseRegionProcedure 953f259f29896493f17802c953dd7fd4, server=1ba5af43fd99,36251,1732710241631 in 1.5910 sec 2024-11-27T12:24:39,399 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=29, resume processing ppid=28 2024-11-27T12:24:39,399 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, ppid=28, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=953f259f29896493f17802c953dd7fd4, UNASSIGN in 1.5980 sec 2024-11-27T12:24:39,401 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=28, resume processing ppid=27 2024-11-27T12:24:39,401 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, ppid=27, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.6050 sec 2024-11-27T12:24:39,402 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732710279402"}]},"ts":"1732710279402"} 2024-11-27T12:24:39,403 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-27T12:24:39,403 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-27T12:24:39,405 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.6180 sec 2024-11-27T12:24:39,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-27T12:24:39,940 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-27T12:24:39,948 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36215 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-11-27T12:24:39,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36215 {}] procedure2.ProcedureExecutor(1139): Stored pid=31, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-27T12:24:39,953 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=31, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-27T12:24:39,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-11-27T12:24:39,955 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=31, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-27T12:24:39,959 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4 2024-11-27T12:24:39,963 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/A, FileablePath, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/B, FileablePath, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/C, FileablePath, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/recovered.edits] 2024-11-27T12:24:39,966 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/A/4c8fc184ea4241f69df936d41192dee9 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/A/4c8fc184ea4241f69df936d41192dee9 2024-11-27T12:24:39,968 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/A/54bc72d14895431e878429cb0d578741 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/A/54bc72d14895431e878429cb0d578741 2024-11-27T12:24:39,969 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/A/a1c1a891d7b649a0b884e8722fba2c03 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/A/a1c1a891d7b649a0b884e8722fba2c03 2024-11-27T12:24:39,971 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/B/1e8748bae58c435d8b3c2ecea29a4232 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/B/1e8748bae58c435d8b3c2ecea29a4232 2024-11-27T12:24:39,973 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/B/6e53f9d45212439db2e7455af6794c05 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/B/6e53f9d45212439db2e7455af6794c05 2024-11-27T12:24:39,974 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/B/71af1a70dd2448f3b0167fe2090a900d to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/B/71af1a70dd2448f3b0167fe2090a900d 2024-11-27T12:24:39,976 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/C/61b84f6a65954df08a8e5c483cf3151c to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/C/61b84f6a65954df08a8e5c483cf3151c 2024-11-27T12:24:39,978 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/C/b754f52484ab4b298d7fc1ae0687d5a6 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/C/b754f52484ab4b298d7fc1ae0687d5a6 2024-11-27T12:24:39,979 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/C/cb3b0a9433764c09b3c82808a3908953 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/C/cb3b0a9433764c09b3c82808a3908953 2024-11-27T12:24:39,982 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/recovered.edits/479.seqid to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4/recovered.edits/479.seqid 2024-11-27T12:24:39,982 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/953f259f29896493f17802c953dd7fd4 2024-11-27T12:24:39,982 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-27T12:24:39,988 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=31, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-27T12:24:39,992 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36251 {}] util.ReflectedFunctionCache(97): Populated cache for org.apache.hadoop.hbase.filter.KeyOnlyFilter in 0ms 2024-11-27T12:24:39,997 WARN [PEWorker-2 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-27T12:24:40,029 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-27T12:24:40,031 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=31, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-27T12:24:40,031 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-27T12:24:40,032 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732710280031"}]},"ts":"9223372036854775807"} 2024-11-27T12:24:40,036 INFO [PEWorker-2 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-27T12:24:40,036 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 953f259f29896493f17802c953dd7fd4, NAME => 'TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4.', STARTKEY => '', ENDKEY => ''}] 2024-11-27T12:24:40,037 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-27T12:24:40,037 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732710280037"}]},"ts":"9223372036854775807"} 2024-11-27T12:24:40,040 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-27T12:24:40,041 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(133): Finished pid=31, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-27T12:24:40,043 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 93 msec 2024-11-27T12:24:40,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-11-27T12:24:40,059 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-27T12:24:40,059 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-27T12:24:40,073 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMixedAtomicity Thread=237 (was 211) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-14 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-15 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-12 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_CLOSE_REGION-regionserver/1ba5af43fd99:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_REGION-regionserver/1ba5af43fd99:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-9 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-16 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-11 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-10 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS:0;1ba5af43fd99:36251-shortCompactions-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.PriorityBlockingQueue.take(PriorityBlockingQueue.java:535) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1180310523_22 at /127.0.0.1:41794 [Waiting for operation #13] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-13 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=445 (was 441) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=402 (was 386) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=8877 (was 9896) 2024-11-27T12:24:40,085 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMobMixedAtomicity Thread=237, OpenFileDescriptor=445, MaxFileDescriptor=1048576, SystemLoadAverage=402, ProcessCount=11, AvailableMemoryMB=8876 2024-11-27T12:24:40,086 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36215 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-27T12:24:40,087 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36215 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-27T12:24:40,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36215 {}] procedure2.ProcedureExecutor(1139): Stored pid=32, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-27T12:24:40,090 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=32, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-27T12:24:40,090 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:40,090 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36215 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 32 2024-11-27T12:24:40,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-27T12:24:40,092 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=32, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-27T12:24:40,100 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073741936_1112 (size=959) 2024-11-27T12:24:40,102 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 5dcfd84eb369d011a0951d03c984f736, NAME => 'TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c 2024-11-27T12:24:40,126 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073741937_1113 (size=53) 2024-11-27T12:24:40,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-27T12:24:40,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-27T12:24:40,527 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T12:24:40,527 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 5dcfd84eb369d011a0951d03c984f736, disabling compactions & flushes 2024-11-27T12:24:40,527 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736. 2024-11-27T12:24:40,527 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736. 2024-11-27T12:24:40,528 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736. after waiting 0 ms 2024-11-27T12:24:40,528 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736. 2024-11-27T12:24:40,528 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736. 2024-11-27T12:24:40,528 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 5dcfd84eb369d011a0951d03c984f736: Waiting for close lock at 1732710280527Disabling compacts and flushes for region at 1732710280527Disabling writes for close at 1732710280528 (+1 ms)Writing region close event to WAL at 1732710280528Closed at 1732710280528 2024-11-27T12:24:40,530 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=32, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-27T12:24:40,530 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732710280530"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732710280530"}]},"ts":"1732710280530"} 2024-11-27T12:24:40,534 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-27T12:24:40,535 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=32, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-27T12:24:40,536 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732710280535"}]},"ts":"1732710280535"} 2024-11-27T12:24:40,539 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-27T12:24:40,539 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5dcfd84eb369d011a0951d03c984f736, ASSIGN}] 2024-11-27T12:24:40,541 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5dcfd84eb369d011a0951d03c984f736, ASSIGN 2024-11-27T12:24:40,543 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=5dcfd84eb369d011a0951d03c984f736, ASSIGN; state=OFFLINE, location=1ba5af43fd99,36251,1732710241631; forceNewPlan=false, retain=false 2024-11-27T12:24:40,693 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=33 updating hbase:meta row=5dcfd84eb369d011a0951d03c984f736, regionState=OPENING, regionLocation=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:40,696 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=5dcfd84eb369d011a0951d03c984f736, ASSIGN because future has completed 2024-11-27T12:24:40,696 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=34, ppid=33, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631}] 2024-11-27T12:24:40,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-27T12:24:40,853 INFO [RS_OPEN_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736. 2024-11-27T12:24:40,853 DEBUG [RS_OPEN_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7752): Opening region: {ENCODED => 5dcfd84eb369d011a0951d03c984f736, NAME => 'TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736.', STARTKEY => '', ENDKEY => ''} 2024-11-27T12:24:40,853 DEBUG [RS_OPEN_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 5dcfd84eb369d011a0951d03c984f736 2024-11-27T12:24:40,853 DEBUG [RS_OPEN_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T12:24:40,854 DEBUG [RS_OPEN_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7794): checking encryption for 5dcfd84eb369d011a0951d03c984f736 2024-11-27T12:24:40,854 DEBUG [RS_OPEN_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7797): checking classloading for 5dcfd84eb369d011a0951d03c984f736 2024-11-27T12:24:40,856 INFO [StoreOpener-5dcfd84eb369d011a0951d03c984f736-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 5dcfd84eb369d011a0951d03c984f736 2024-11-27T12:24:40,859 INFO [StoreOpener-5dcfd84eb369d011a0951d03c984f736-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5dcfd84eb369d011a0951d03c984f736 columnFamilyName A 2024-11-27T12:24:40,860 DEBUG [StoreOpener-5dcfd84eb369d011a0951d03c984f736-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:40,860 INFO [StoreOpener-5dcfd84eb369d011a0951d03c984f736-1 {}] regionserver.HStore(327): Store=5dcfd84eb369d011a0951d03c984f736/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T12:24:40,861 INFO [StoreOpener-5dcfd84eb369d011a0951d03c984f736-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 5dcfd84eb369d011a0951d03c984f736 2024-11-27T12:24:40,863 INFO [StoreOpener-5dcfd84eb369d011a0951d03c984f736-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5dcfd84eb369d011a0951d03c984f736 columnFamilyName B 2024-11-27T12:24:40,863 DEBUG [StoreOpener-5dcfd84eb369d011a0951d03c984f736-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:40,864 INFO [StoreOpener-5dcfd84eb369d011a0951d03c984f736-1 {}] regionserver.HStore(327): Store=5dcfd84eb369d011a0951d03c984f736/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T12:24:40,865 INFO [StoreOpener-5dcfd84eb369d011a0951d03c984f736-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 5dcfd84eb369d011a0951d03c984f736 2024-11-27T12:24:40,916 INFO [StoreOpener-5dcfd84eb369d011a0951d03c984f736-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5dcfd84eb369d011a0951d03c984f736 columnFamilyName C 2024-11-27T12:24:40,916 DEBUG [StoreOpener-5dcfd84eb369d011a0951d03c984f736-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:40,917 INFO [StoreOpener-5dcfd84eb369d011a0951d03c984f736-1 {}] regionserver.HStore(327): Store=5dcfd84eb369d011a0951d03c984f736/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T12:24:40,917 DEBUG [RS_OPEN_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1038): replaying wal for 5dcfd84eb369d011a0951d03c984f736 2024-11-27T12:24:40,918 DEBUG [RS_OPEN_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736 2024-11-27T12:24:40,919 DEBUG [RS_OPEN_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736 2024-11-27T12:24:40,921 DEBUG [RS_OPEN_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1048): stopping wal replay for 5dcfd84eb369d011a0951d03c984f736 2024-11-27T12:24:40,921 DEBUG [RS_OPEN_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1060): Cleaning up temporary data for 5dcfd84eb369d011a0951d03c984f736 2024-11-27T12:24:40,922 DEBUG [RS_OPEN_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-27T12:24:40,925 DEBUG [RS_OPEN_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1093): writing seq id for 5dcfd84eb369d011a0951d03c984f736 2024-11-27T12:24:40,929 DEBUG [RS_OPEN_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-27T12:24:40,930 INFO [RS_OPEN_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1114): Opened 5dcfd84eb369d011a0951d03c984f736; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72506609, jitterRate=0.08043266832828522}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-27T12:24:40,930 DEBUG [RS_OPEN_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 5dcfd84eb369d011a0951d03c984f736 2024-11-27T12:24:40,932 DEBUG [RS_OPEN_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1006): Region open journal for 5dcfd84eb369d011a0951d03c984f736: Running coprocessor pre-open hook at 1732710280854Writing region info on filesystem at 1732710280854Initializing all the Stores at 1732710280855 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732710280855Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732710280855Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732710280855Cleaning up temporary data from old regions at 1732710280921 (+66 ms)Running coprocessor post-open hooks at 1732710280930 (+9 ms)Region opened successfully at 1732710280932 (+2 ms) 2024-11-27T12:24:40,933 INFO [RS_OPEN_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., pid=34, masterSystemTime=1732710280849 2024-11-27T12:24:40,935 DEBUG [RS_OPEN_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736. 2024-11-27T12:24:40,935 INFO [RS_OPEN_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736. 2024-11-27T12:24:40,935 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=33 updating hbase:meta row=5dcfd84eb369d011a0951d03c984f736, regionState=OPEN, openSeqNum=2, regionLocation=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:40,937 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=34, ppid=33, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 because future has completed 2024-11-27T12:24:40,940 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=34, resume processing ppid=33 2024-11-27T12:24:40,940 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=34, ppid=33, state=SUCCESS, hasLock=false; OpenRegionProcedure 5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 in 242 msec 2024-11-27T12:24:40,942 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=33, resume processing ppid=32 2024-11-27T12:24:40,942 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5dcfd84eb369d011a0951d03c984f736, ASSIGN in 401 msec 2024-11-27T12:24:40,943 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=32, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-27T12:24:40,944 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732710280943"}]},"ts":"1732710280943"} 2024-11-27T12:24:40,945 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-27T12:24:40,946 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=32, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-27T12:24:40,948 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 859 msec 2024-11-27T12:24:41,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-27T12:24:41,229 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-27T12:24:41,231 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@794f42f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T12:24:41,232 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1ba5af43fd99,36215,-1 for getting cluster id 2024-11-27T12:24:41,232 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T12:24:41,235 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '200978c2-2c41-4f93-b0b5-127ea1024f37' 2024-11-27T12:24:41,236 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T12:24:41,237 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "200978c2-2c41-4f93-b0b5-127ea1024f37" 2024-11-27T12:24:41,237 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@c61c610, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T12:24:41,237 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1ba5af43fd99,36215,-1] 2024-11-27T12:24:41,238 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T12:24:41,238 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T12:24:41,240 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57632, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T12:24:41,241 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@63bdc2aa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T12:24:41,241 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T12:24:41,242 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1ba5af43fd99,36251,1732710241631, seqNum=-1] 2024-11-27T12:24:41,243 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T12:24:41,244 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:52814, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T12:24:41,246 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-27T12:24:41,247 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncConnectionImpl(321): The fetched master address is 1ba5af43fd99,36215,1732710240860 2024-11-27T12:24:41,247 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@47569338 2024-11-27T12:24:41,247 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-27T12:24:41,248 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57638, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-27T12:24:41,254 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36215 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-27T12:24:41,254 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36215 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.3 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-27T12:24:41,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36215 {}] procedure2.ProcedureExecutor(1139): Stored pid=35, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-27T12:24:41,275 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073741938_1114 (size=995) 2024-11-27T12:24:41,677 DEBUG [PEWorker-2 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.959 2024-11-27T12:24:41,677 INFO [PEWorker-2 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.995 2024-11-27T12:24:41,683 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=36, ppid=35, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-27T12:24:41,692 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5dcfd84eb369d011a0951d03c984f736, REOPEN/MOVE}] 2024-11-27T12:24:41,693 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5dcfd84eb369d011a0951d03c984f736, REOPEN/MOVE 2024-11-27T12:24:41,694 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=37 updating hbase:meta row=5dcfd84eb369d011a0951d03c984f736, regionState=CLOSING, regionLocation=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:41,696 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=5dcfd84eb369d011a0951d03c984f736, REOPEN/MOVE because future has completed 2024-11-27T12:24:41,697 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-27T12:24:41,697 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=38, ppid=37, state=RUNNABLE, hasLock=false; CloseRegionProcedure 5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631}] 2024-11-27T12:24:41,850 INFO [RS_CLOSE_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] handler.UnassignRegionHandler(122): Close 5dcfd84eb369d011a0951d03c984f736 2024-11-27T12:24:41,850 DEBUG [RS_CLOSE_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-27T12:24:41,850 DEBUG [RS_CLOSE_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1722): Closing 5dcfd84eb369d011a0951d03c984f736, disabling compactions & flushes 2024-11-27T12:24:41,850 INFO [RS_CLOSE_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736. 2024-11-27T12:24:41,850 DEBUG [RS_CLOSE_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736. 2024-11-27T12:24:41,850 DEBUG [RS_CLOSE_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736. after waiting 0 ms 2024-11-27T12:24:41,850 DEBUG [RS_CLOSE_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736. 2024-11-27T12:24:41,854 DEBUG [RS_CLOSE_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-27T12:24:41,855 INFO [RS_CLOSE_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736. 2024-11-27T12:24:41,855 DEBUG [RS_CLOSE_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1676): Region close journal for 5dcfd84eb369d011a0951d03c984f736: Waiting for close lock at 1732710281850Running coprocessor pre-close hooks at 1732710281850Disabling compacts and flushes for region at 1732710281850Disabling writes for close at 1732710281850Writing region close event to WAL at 1732710281851 (+1 ms)Running coprocessor post-close hooks at 1732710281855 (+4 ms)Closed at 1732710281855 2024-11-27T12:24:41,855 WARN [RS_CLOSE_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegionServer(3299): Not adding moved region record: 5dcfd84eb369d011a0951d03c984f736 to self. 2024-11-27T12:24:41,857 INFO [RS_CLOSE_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] handler.UnassignRegionHandler(157): Closed 5dcfd84eb369d011a0951d03c984f736 2024-11-27T12:24:41,858 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=37 updating hbase:meta row=5dcfd84eb369d011a0951d03c984f736, regionState=CLOSED 2024-11-27T12:24:41,859 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=38, ppid=37, state=RUNNABLE, hasLock=false; CloseRegionProcedure 5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 because future has completed 2024-11-27T12:24:41,862 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=38, resume processing ppid=37 2024-11-27T12:24:41,862 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=38, ppid=37, state=SUCCESS, hasLock=false; CloseRegionProcedure 5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 in 164 msec 2024-11-27T12:24:41,863 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=5dcfd84eb369d011a0951d03c984f736, REOPEN/MOVE; state=CLOSED, location=1ba5af43fd99,36251,1732710241631; forceNewPlan=false, retain=true 2024-11-27T12:24:42,014 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=37 updating hbase:meta row=5dcfd84eb369d011a0951d03c984f736, regionState=OPENING, regionLocation=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:42,017 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=5dcfd84eb369d011a0951d03c984f736, REOPEN/MOVE because future has completed 2024-11-27T12:24:42,018 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=39, ppid=37, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631}] 2024-11-27T12:24:42,175 INFO [RS_OPEN_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736. 2024-11-27T12:24:42,175 DEBUG [RS_OPEN_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7752): Opening region: {ENCODED => 5dcfd84eb369d011a0951d03c984f736, NAME => 'TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736.', STARTKEY => '', ENDKEY => ''} 2024-11-27T12:24:42,175 DEBUG [RS_OPEN_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 5dcfd84eb369d011a0951d03c984f736 2024-11-27T12:24:42,175 DEBUG [RS_OPEN_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T12:24:42,176 DEBUG [RS_OPEN_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7794): checking encryption for 5dcfd84eb369d011a0951d03c984f736 2024-11-27T12:24:42,176 DEBUG [RS_OPEN_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7797): checking classloading for 5dcfd84eb369d011a0951d03c984f736 2024-11-27T12:24:42,181 INFO [StoreOpener-5dcfd84eb369d011a0951d03c984f736-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 5dcfd84eb369d011a0951d03c984f736 2024-11-27T12:24:42,189 INFO [StoreOpener-5dcfd84eb369d011a0951d03c984f736-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5dcfd84eb369d011a0951d03c984f736 columnFamilyName A 2024-11-27T12:24:42,191 DEBUG [StoreOpener-5dcfd84eb369d011a0951d03c984f736-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:42,191 INFO [StoreOpener-5dcfd84eb369d011a0951d03c984f736-1 {}] regionserver.HStore(327): Store=5dcfd84eb369d011a0951d03c984f736/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T12:24:42,192 INFO [StoreOpener-5dcfd84eb369d011a0951d03c984f736-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 5dcfd84eb369d011a0951d03c984f736 2024-11-27T12:24:42,193 INFO [StoreOpener-5dcfd84eb369d011a0951d03c984f736-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5dcfd84eb369d011a0951d03c984f736 columnFamilyName B 2024-11-27T12:24:42,193 DEBUG [StoreOpener-5dcfd84eb369d011a0951d03c984f736-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:42,193 INFO [StoreOpener-5dcfd84eb369d011a0951d03c984f736-1 {}] regionserver.HStore(327): Store=5dcfd84eb369d011a0951d03c984f736/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T12:24:42,193 INFO [StoreOpener-5dcfd84eb369d011a0951d03c984f736-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 5dcfd84eb369d011a0951d03c984f736 2024-11-27T12:24:42,194 INFO [StoreOpener-5dcfd84eb369d011a0951d03c984f736-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5dcfd84eb369d011a0951d03c984f736 columnFamilyName C 2024-11-27T12:24:42,194 DEBUG [StoreOpener-5dcfd84eb369d011a0951d03c984f736-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:42,194 INFO [StoreOpener-5dcfd84eb369d011a0951d03c984f736-1 {}] regionserver.HStore(327): Store=5dcfd84eb369d011a0951d03c984f736/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T12:24:42,194 DEBUG [RS_OPEN_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1038): replaying wal for 5dcfd84eb369d011a0951d03c984f736 2024-11-27T12:24:42,195 DEBUG [RS_OPEN_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736 2024-11-27T12:24:42,196 DEBUG [RS_OPEN_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736 2024-11-27T12:24:42,197 DEBUG [RS_OPEN_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1048): stopping wal replay for 5dcfd84eb369d011a0951d03c984f736 2024-11-27T12:24:42,197 DEBUG [RS_OPEN_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1060): Cleaning up temporary data for 5dcfd84eb369d011a0951d03c984f736 2024-11-27T12:24:42,198 DEBUG [RS_OPEN_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-27T12:24:42,199 DEBUG [RS_OPEN_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1093): writing seq id for 5dcfd84eb369d011a0951d03c984f736 2024-11-27T12:24:42,200 INFO [RS_OPEN_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1114): Opened 5dcfd84eb369d011a0951d03c984f736; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60513739, jitterRate=-0.09827502071857452}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-27T12:24:42,200 DEBUG [RS_OPEN_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 5dcfd84eb369d011a0951d03c984f736 2024-11-27T12:24:42,201 DEBUG [RS_OPEN_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1006): Region open journal for 5dcfd84eb369d011a0951d03c984f736: Running coprocessor pre-open hook at 1732710282176Writing region info on filesystem at 1732710282176Initializing all the Stores at 1732710282177 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732710282177Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732710282178 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732710282178Cleaning up temporary data from old regions at 1732710282197 (+19 ms)Running coprocessor post-open hooks at 1732710282200 (+3 ms)Region opened successfully at 1732710282201 (+1 ms) 2024-11-27T12:24:42,202 INFO [RS_OPEN_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., pid=39, masterSystemTime=1732710282171 2024-11-27T12:24:42,204 DEBUG [RS_OPEN_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736. 2024-11-27T12:24:42,204 INFO [RS_OPEN_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736. 2024-11-27T12:24:42,205 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=37 updating hbase:meta row=5dcfd84eb369d011a0951d03c984f736, regionState=OPEN, openSeqNum=5, regionLocation=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:42,206 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=39, ppid=37, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 because future has completed 2024-11-27T12:24:42,209 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=39, resume processing ppid=37 2024-11-27T12:24:42,209 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=39, ppid=37, state=SUCCESS, hasLock=false; OpenRegionProcedure 5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 in 189 msec 2024-11-27T12:24:42,211 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=37, resume processing ppid=36 2024-11-27T12:24:42,211 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=37, ppid=36, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5dcfd84eb369d011a0951d03c984f736, REOPEN/MOVE in 517 msec 2024-11-27T12:24:42,215 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=36, resume processing ppid=35 2024-11-27T12:24:42,215 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=36, ppid=35, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 530 msec 2024-11-27T12:24:42,219 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=35, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 961 msec 2024-11-27T12:24:42,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=35 2024-11-27T12:24:42,221 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-27T12:24:42,222 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1d7f027b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T12:24:42,222 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1ba5af43fd99,36215,-1 for getting cluster id 2024-11-27T12:24:42,222 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T12:24:42,223 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '200978c2-2c41-4f93-b0b5-127ea1024f37' 2024-11-27T12:24:42,223 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T12:24:42,224 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "200978c2-2c41-4f93-b0b5-127ea1024f37" 2024-11-27T12:24:42,224 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3ffc4a71, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T12:24:42,224 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1ba5af43fd99,36215,-1] 2024-11-27T12:24:42,224 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T12:24:42,224 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T12:24:42,225 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40590, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T12:24:42,226 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3146dbb1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T12:24:42,227 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@34696a8a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T12:24:42,227 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1ba5af43fd99,36215,-1 for getting cluster id 2024-11-27T12:24:42,227 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T12:24:42,228 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '200978c2-2c41-4f93-b0b5-127ea1024f37' 2024-11-27T12:24:42,228 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T12:24:42,228 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "200978c2-2c41-4f93-b0b5-127ea1024f37" 2024-11-27T12:24:42,229 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@145ca288, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T12:24:42,229 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1ba5af43fd99,36215,-1] 2024-11-27T12:24:42,229 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T12:24:42,229 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T12:24:42,230 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40608, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T12:24:42,231 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6c9d3125, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T12:24:42,231 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@35f79bd2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T12:24:42,232 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1ba5af43fd99,36215,-1 for getting cluster id 2024-11-27T12:24:42,232 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T12:24:42,233 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '200978c2-2c41-4f93-b0b5-127ea1024f37' 2024-11-27T12:24:42,233 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T12:24:42,233 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "200978c2-2c41-4f93-b0b5-127ea1024f37" 2024-11-27T12:24:42,233 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5648fe28, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T12:24:42,233 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1ba5af43fd99,36215,-1] 2024-11-27T12:24:42,233 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T12:24:42,233 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T12:24:42,234 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40614, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T12:24:42,235 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1d75258c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T12:24:42,236 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@55b0628d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T12:24:42,236 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1ba5af43fd99,36215,-1 for getting cluster id 2024-11-27T12:24:42,236 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T12:24:42,237 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '200978c2-2c41-4f93-b0b5-127ea1024f37' 2024-11-27T12:24:42,237 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T12:24:42,237 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "200978c2-2c41-4f93-b0b5-127ea1024f37" 2024-11-27T12:24:42,237 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@245dc231, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T12:24:42,237 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1ba5af43fd99,36215,-1] 2024-11-27T12:24:42,238 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T12:24:42,238 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T12:24:42,239 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40634, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T12:24:42,239 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@49a8c38e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T12:24:42,240 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@430b62f8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T12:24:42,240 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1ba5af43fd99,36215,-1 for getting cluster id 2024-11-27T12:24:42,240 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T12:24:42,241 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '200978c2-2c41-4f93-b0b5-127ea1024f37' 2024-11-27T12:24:42,241 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T12:24:42,241 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "200978c2-2c41-4f93-b0b5-127ea1024f37" 2024-11-27T12:24:42,242 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f5df044, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T12:24:42,242 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1ba5af43fd99,36215,-1] 2024-11-27T12:24:42,242 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T12:24:42,242 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T12:24:42,243 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40650, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T12:24:42,244 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3ed4f392, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T12:24:42,245 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4f2b1a52, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T12:24:42,245 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1ba5af43fd99,36215,-1 for getting cluster id 2024-11-27T12:24:42,245 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T12:24:42,246 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '200978c2-2c41-4f93-b0b5-127ea1024f37' 2024-11-27T12:24:42,246 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T12:24:42,246 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "200978c2-2c41-4f93-b0b5-127ea1024f37" 2024-11-27T12:24:42,246 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7924ba72, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T12:24:42,247 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1ba5af43fd99,36215,-1] 2024-11-27T12:24:42,247 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T12:24:42,247 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T12:24:42,248 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40672, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T12:24:42,249 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@57832cf6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T12:24:42,250 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@695519cf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T12:24:42,250 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1ba5af43fd99,36215,-1 for getting cluster id 2024-11-27T12:24:42,250 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T12:24:42,251 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '200978c2-2c41-4f93-b0b5-127ea1024f37' 2024-11-27T12:24:42,252 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T12:24:42,252 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "200978c2-2c41-4f93-b0b5-127ea1024f37" 2024-11-27T12:24:42,252 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6b837274, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T12:24:42,252 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1ba5af43fd99,36215,-1] 2024-11-27T12:24:42,252 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T12:24:42,253 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T12:24:42,254 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40696, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T12:24:42,255 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@40682de9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T12:24:42,255 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@27c9c0b6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T12:24:42,255 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1ba5af43fd99,36215,-1 for getting cluster id 2024-11-27T12:24:42,256 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T12:24:42,257 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '200978c2-2c41-4f93-b0b5-127ea1024f37' 2024-11-27T12:24:42,257 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T12:24:42,257 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "200978c2-2c41-4f93-b0b5-127ea1024f37" 2024-11-27T12:24:42,258 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4444597e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T12:24:42,258 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1ba5af43fd99,36215,-1] 2024-11-27T12:24:42,258 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T12:24:42,258 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T12:24:42,259 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40708, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T12:24:42,260 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@50c6f230, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T12:24:42,261 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3d019e2a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T12:24:42,261 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1ba5af43fd99,36215,-1 for getting cluster id 2024-11-27T12:24:42,261 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T12:24:42,262 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '200978c2-2c41-4f93-b0b5-127ea1024f37' 2024-11-27T12:24:42,262 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T12:24:42,262 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "200978c2-2c41-4f93-b0b5-127ea1024f37" 2024-11-27T12:24:42,263 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@339a0fec, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T12:24:42,263 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1ba5af43fd99,36215,-1] 2024-11-27T12:24:42,263 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T12:24:42,263 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T12:24:42,264 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40730, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T12:24:42,265 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@68ba1929, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T12:24:42,266 DEBUG [Thread-587 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T12:24:42,266 DEBUG [Thread-591 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T12:24:42,266 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1ba5af43fd99,36251,1732710241631, seqNum=-1] 2024-11-27T12:24:42,266 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1ba5af43fd99,36251,1732710241631, seqNum=-1] 2024-11-27T12:24:42,267 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T12:24:42,267 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T12:24:42,268 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44010, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T12:24:42,268 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44012, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T12:24:42,273 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5] 2024-11-27T12:24:42,273 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36215 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-27T12:24:42,274 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5] 2024-11-27T12:24:42,274 DEBUG [Thread-598 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T12:24:42,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36215 {}] procedure2.ProcedureExecutor(1139): Stored pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees 2024-11-27T12:24:42,276 DEBUG [Thread-593 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T12:24:42,277 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T12:24:42,277 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1ba5af43fd99,36251,1732710241631, seqNum=-1] 2024-11-27T12:24:42,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-27T12:24:42,277 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T12:24:42,277 DEBUG [Thread-598 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1ba5af43fd99,36251,1732710241631, seqNum=-1] 2024-11-27T12:24:42,278 DEBUG [Thread-598 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T12:24:42,278 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T12:24:42,278 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44028, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T12:24:42,278 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=41, ppid=40, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T12:24:42,279 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44040, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T12:24:42,282 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5] 2024-11-27T12:24:42,283 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5] 2024-11-27T12:24:42,283 DEBUG [Thread-600 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T12:24:42,284 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1ba5af43fd99,36251,1732710241631, seqNum=-1] 2024-11-27T12:24:42,284 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T12:24:42,285 DEBUG [Thread-604 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T12:24:42,285 DEBUG [Thread-589 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T12:24:42,286 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44046, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T12:24:42,286 DEBUG [Thread-595 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T12:24:42,287 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1ba5af43fd99,36251,1732710241631, seqNum=-1] 2024-11-27T12:24:42,287 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1ba5af43fd99,36251,1732710241631, seqNum=-1] 2024-11-27T12:24:42,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(8855): Flush requested on 5dcfd84eb369d011a0951d03c984f736 2024-11-27T12:24:42,287 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T12:24:42,287 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T12:24:42,288 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1ba5af43fd99,36251,1732710241631, seqNum=-1] 2024-11-27T12:24:42,288 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T12:24:42,288 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5dcfd84eb369d011a0951d03c984f736 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-27T12:24:42,288 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T12:24:42,288 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44052, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T12:24:42,288 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44060, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T12:24:42,289 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5] 2024-11-27T12:24:42,290 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44064, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T12:24:42,292 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5] 2024-11-27T12:24:42,292 DEBUG [Thread-602 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T12:24:42,293 DEBUG [Thread-602 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1ba5af43fd99,36251,1732710241631, seqNum=-1] 2024-11-27T12:24:42,294 DEBUG [Thread-602 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T12:24:42,296 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5] 2024-11-27T12:24:42,297 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5] 2024-11-27T12:24:42,297 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44078, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T12:24:42,300 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5] 2024-11-27T12:24:42,340 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127f1662ebf04ff4ffe990ddda36ea12e0f_5dcfd84eb369d011a0951d03c984f736 is 50, key is test_row_0/A:col10/1732710282287/Put/seqid=0 2024-11-27T12:24:42,352 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:42,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44052 deadline: 1732710342338, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:42,353 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:42,353 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:42,353 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:42,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44064 deadline: 1732710342342, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:42,353 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:24:42,354 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:42,354 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:42,354 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:24:42,362 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073741939_1115 (size=14594) 2024-11-27T12:24:42,363 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:42,369 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127f1662ebf04ff4ffe990ddda36ea12e0f_5dcfd84eb369d011a0951d03c984f736 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127f1662ebf04ff4ffe990ddda36ea12e0f_5dcfd84eb369d011a0951d03c984f736 2024-11-27T12:24:42,370 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/.tmp/A/087bb11cf4cb49c1b6f5f185314181d9, store: [table=TestAcidGuarantees family=A region=5dcfd84eb369d011a0951d03c984f736] 2024-11-27T12:24:42,371 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:42,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44012 deadline: 1732710342353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:42,374 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:42,374 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:42,375 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:42,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44028 deadline: 1732710342358, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:42,375 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:42,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44010 deadline: 1732710342358, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:42,375 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:24:42,375 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:42,375 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:42,375 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:24:42,376 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:42,376 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:42,376 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:24:42,379 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/.tmp/A/087bb11cf4cb49c1b6f5f185314181d9 is 175, key is test_row_0/A:col10/1732710282287/Put/seqid=0 2024-11-27T12:24:42,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-27T12:24:42,401 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073741940_1116 (size=39549) 2024-11-27T12:24:42,436 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-27T12:24:42,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736. 2024-11-27T12:24:42,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736. as already flushing 2024-11-27T12:24:42,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736. 2024-11-27T12:24:42,436 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 5dcfd84eb369d011a0951d03c984f736, NAME => 'TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:24:42,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 5dcfd84eb369d011a0951d03c984f736, NAME => 'TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:24:42,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5dcfd84eb369d011a0951d03c984f736, NAME => 'TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5dcfd84eb369d011a0951d03c984f736, NAME => 'TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:24:42,460 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:42,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44052 deadline: 1732710342459, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:42,461 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:42,461 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:42,461 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:42,461 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:24:42,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44064 deadline: 1732710342459, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:42,461 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:42,461 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:42,461 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:24:42,485 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:42,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44028 deadline: 1732710342479, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:42,485 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:42,485 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:42,485 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:42,485 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:24:42,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44012 deadline: 1732710342479, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:42,486 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:42,486 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:42,486 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:24:42,494 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:42,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44010 deadline: 1732710342488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:42,495 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:42,495 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:42,495 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:24:42,590 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-27T12:24:42,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736. 2024-11-27T12:24:42,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736. as already flushing 2024-11-27T12:24:42,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736. 2024-11-27T12:24:42,590 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 5dcfd84eb369d011a0951d03c984f736, NAME => 'TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:24:42,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 5dcfd84eb369d011a0951d03c984f736, NAME => 'TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:24:42,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5dcfd84eb369d011a0951d03c984f736, NAME => 'TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5dcfd84eb369d011a0951d03c984f736, NAME => 'TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:24:42,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-27T12:24:42,669 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:42,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44052 deadline: 1732710342668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:42,670 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:42,670 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:42,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44064 deadline: 1732710342668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:42,670 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:42,670 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:24:42,670 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:42,670 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:42,670 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:24:42,689 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:42,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44012 deadline: 1732710342688, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:42,690 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:42,690 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:42,690 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:24:42,698 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:42,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44028 deadline: 1732710342698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:42,699 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:42,699 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:42,699 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:24:42,699 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:42,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44010 deadline: 1732710342698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:42,700 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:42,700 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:42,700 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:24:42,747 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-27T12:24:42,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736. 2024-11-27T12:24:42,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736. as already flushing 2024-11-27T12:24:42,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736. 2024-11-27T12:24:42,748 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 5dcfd84eb369d011a0951d03c984f736, NAME => 'TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:24:42,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 5dcfd84eb369d011a0951d03c984f736, NAME => 'TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:24:42,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5dcfd84eb369d011a0951d03c984f736, NAME => 'TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5dcfd84eb369d011a0951d03c984f736, NAME => 'TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:24:42,806 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=17, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/.tmp/A/087bb11cf4cb49c1b6f5f185314181d9 2024-11-27T12:24:42,841 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/.tmp/B/583ff06821a042adba3819a0e06cc3e4 is 50, key is test_row_0/B:col10/1732710282287/Put/seqid=0 2024-11-27T12:24:42,850 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073741941_1117 (size=12001) 2024-11-27T12:24:42,903 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-27T12:24:42,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736. 2024-11-27T12:24:42,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736. as already flushing 2024-11-27T12:24:42,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736. 2024-11-27T12:24:42,904 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 5dcfd84eb369d011a0951d03c984f736, NAME => 'TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:24:42,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 5dcfd84eb369d011a0951d03c984f736, NAME => 'TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:24:42,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5dcfd84eb369d011a0951d03c984f736, NAME => 'TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5dcfd84eb369d011a0951d03c984f736, NAME => 'TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:24:42,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-27T12:24:42,982 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:42,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44052 deadline: 1732710342978, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:42,982 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:42,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44064 deadline: 1732710342978, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:42,983 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:42,983 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:42,983 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:42,983 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:42,983 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:24:42,983 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:24:42,999 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:42,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44012 deadline: 1732710342998, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:43,000 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:43,000 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:43,000 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:24:43,010 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:43,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44010 deadline: 1732710343009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:43,012 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:43,012 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:43,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44028 deadline: 1732710343009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:43,012 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:43,012 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:24:43,013 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:43,013 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:43,013 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:24:43,057 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-27T12:24:43,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736. 2024-11-27T12:24:43,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736. as already flushing 2024-11-27T12:24:43,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736. 2024-11-27T12:24:43,057 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 5dcfd84eb369d011a0951d03c984f736, NAME => 'TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:24:43,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 5dcfd84eb369d011a0951d03c984f736, NAME => 'TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:24:43,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5dcfd84eb369d011a0951d03c984f736, NAME => 'TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5dcfd84eb369d011a0951d03c984f736, NAME => 'TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:24:43,211 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-27T12:24:43,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736. 2024-11-27T12:24:43,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736. as already flushing 2024-11-27T12:24:43,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736. 2024-11-27T12:24:43,211 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 5dcfd84eb369d011a0951d03c984f736, NAME => 'TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:24:43,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 5dcfd84eb369d011a0951d03c984f736, NAME => 'TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:24:43,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5dcfd84eb369d011a0951d03c984f736, NAME => 'TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5dcfd84eb369d011a0951d03c984f736, NAME => 'TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:24:43,251 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/.tmp/B/583ff06821a042adba3819a0e06cc3e4 2024-11-27T12:24:43,285 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/.tmp/C/15779297379e4155b260778392f26a78 is 50, key is test_row_0/C:col10/1732710282287/Put/seqid=0 2024-11-27T12:24:43,295 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073741942_1118 (size=12001) 2024-11-27T12:24:43,297 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/.tmp/C/15779297379e4155b260778392f26a78 2024-11-27T12:24:43,304 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/.tmp/A/087bb11cf4cb49c1b6f5f185314181d9 as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/A/087bb11cf4cb49c1b6f5f185314181d9 2024-11-27T12:24:43,313 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/A/087bb11cf4cb49c1b6f5f185314181d9, entries=200, sequenceid=17, filesize=38.6 K 2024-11-27T12:24:43,315 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/.tmp/B/583ff06821a042adba3819a0e06cc3e4 as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/B/583ff06821a042adba3819a0e06cc3e4 2024-11-27T12:24:43,336 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/B/583ff06821a042adba3819a0e06cc3e4, entries=150, sequenceid=17, filesize=11.7 K 2024-11-27T12:24:43,337 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/.tmp/C/15779297379e4155b260778392f26a78 as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/C/15779297379e4155b260778392f26a78 2024-11-27T12:24:43,345 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/C/15779297379e4155b260778392f26a78, entries=150, sequenceid=17, filesize=11.7 K 2024-11-27T12:24:43,346 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=147.60 KB/151140 for 5dcfd84eb369d011a0951d03c984f736 in 1058ms, sequenceid=17, compaction requested=false 2024-11-27T12:24:43,346 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-27T12:24:43,347 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5dcfd84eb369d011a0951d03c984f736: 2024-11-27T12:24:43,364 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-27T12:24:43,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736. 2024-11-27T12:24:43,364 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2902): Flushing 5dcfd84eb369d011a0951d03c984f736 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-27T12:24:43,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127852cec6eede742e7a5018b6919bc5316_5dcfd84eb369d011a0951d03c984f736 is 50, key is test_row_0/A:col10/1732710282348/Put/seqid=0 2024-11-27T12:24:43,392 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073741943_1119 (size=12154) 2024-11-27T12:24:43,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-27T12:24:43,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(8855): Flush requested on 5dcfd84eb369d011a0951d03c984f736 2024-11-27T12:24:43,492 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736. as already flushing 2024-11-27T12:24:43,521 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:43,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44064 deadline: 1732710343514, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:43,522 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:43,522 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:43,522 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:24:43,526 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:43,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44052 deadline: 1732710343516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:43,527 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:43,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44012 deadline: 1732710343518, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:43,527 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:43,527 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:43,527 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:43,527 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:24:43,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44028 deadline: 1732710343518, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:43,528 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:43,528 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:43,528 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:43,528 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:24:43,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44010 deadline: 1732710343518, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:43,528 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:43,528 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:43,528 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:24:43,531 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:43,531 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:43,531 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:24:43,633 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:43,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44064 deadline: 1732710343628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:43,634 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:43,634 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:43,634 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:24:43,639 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:43,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44052 deadline: 1732710343638, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:43,640 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:43,640 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:43,640 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:43,640 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:24:43,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44012 deadline: 1732710343639, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:43,640 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:43,640 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:43,640 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:24:43,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:43,798 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127852cec6eede742e7a5018b6919bc5316_5dcfd84eb369d011a0951d03c984f736 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127852cec6eede742e7a5018b6919bc5316_5dcfd84eb369d011a0951d03c984f736 2024-11-27T12:24:43,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/.tmp/A/f96b021a7716418ca7c9c432ef18b536, store: [table=TestAcidGuarantees family=A region=5dcfd84eb369d011a0951d03c984f736] 2024-11-27T12:24:43,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/.tmp/A/f96b021a7716418ca7c9c432ef18b536 is 175, key is test_row_0/A:col10/1732710282348/Put/seqid=0 2024-11-27T12:24:43,805 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073741944_1120 (size=30955) 2024-11-27T12:24:43,806 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=42, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/.tmp/A/f96b021a7716418ca7c9c432ef18b536 2024-11-27T12:24:43,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/.tmp/B/c3cc05cc3520420491c4ce2d8926e957 is 50, key is test_row_0/B:col10/1732710282348/Put/seqid=0 2024-11-27T12:24:43,837 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073741945_1121 (size=12001) 2024-11-27T12:24:43,840 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:43,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44064 deadline: 1732710343838, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:43,840 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:43,841 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:43,841 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:24:43,851 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:43,851 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:43,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44052 deadline: 1732710343848, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:43,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44012 deadline: 1732710343848, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:43,852 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:43,852 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:43,852 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:43,852 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:43,852 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:24:43,852 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:24:44,149 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:44,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44064 deadline: 1732710344148, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:44,150 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:44,150 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:44,150 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:24:44,160 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:44,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44052 deadline: 1732710344158, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:44,161 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:44,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44012 deadline: 1732710344159, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:44,162 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:44,162 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:44,162 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:24:44,162 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:44,162 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:44,162 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:24:44,238 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=42 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/.tmp/B/c3cc05cc3520420491c4ce2d8926e957 2024-11-27T12:24:44,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/.tmp/C/22e85f9daff94ca59a9ae29b0df314d9 is 50, key is test_row_0/C:col10/1732710282348/Put/seqid=0 2024-11-27T12:24:44,259 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073741946_1122 (size=12001) 2024-11-27T12:24:44,260 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=42 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/.tmp/C/22e85f9daff94ca59a9ae29b0df314d9 2024-11-27T12:24:44,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/.tmp/A/f96b021a7716418ca7c9c432ef18b536 as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/A/f96b021a7716418ca7c9c432ef18b536 2024-11-27T12:24:44,275 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/A/f96b021a7716418ca7c9c432ef18b536, entries=150, sequenceid=42, filesize=30.2 K 2024-11-27T12:24:44,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/.tmp/B/c3cc05cc3520420491c4ce2d8926e957 as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/B/c3cc05cc3520420491c4ce2d8926e957 2024-11-27T12:24:44,285 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/B/c3cc05cc3520420491c4ce2d8926e957, entries=150, sequenceid=42, filesize=11.7 K 2024-11-27T12:24:44,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/.tmp/C/22e85f9daff94ca59a9ae29b0df314d9 as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/C/22e85f9daff94ca59a9ae29b0df314d9 2024-11-27T12:24:44,293 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/C/22e85f9daff94ca59a9ae29b0df314d9, entries=150, sequenceid=42, filesize=11.7 K 2024-11-27T12:24:44,294 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 5dcfd84eb369d011a0951d03c984f736 in 930ms, sequenceid=42, compaction requested=false 2024-11-27T12:24:44,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2603): Flush status journal for 5dcfd84eb369d011a0951d03c984f736: 2024-11-27T12:24:44,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736. 2024-11-27T12:24:44,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=41 2024-11-27T12:24:44,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.HMaster(4169): Remote procedure done, pid=41 2024-11-27T12:24:44,297 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=41, resume processing ppid=40 2024-11-27T12:24:44,297 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=41, ppid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0180 sec 2024-11-27T12:24:44,299 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees in 2.0250 sec 2024-11-27T12:24:44,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-27T12:24:44,429 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T12:24:44,429 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-27T12:24:44,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] procedure2.ProcedureExecutor(1139): Stored pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees 2024-11-27T12:24:44,432 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T12:24:44,434 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T12:24:44,434 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=43, ppid=42, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T12:24:44,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-27T12:24:44,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-27T12:24:44,550 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T12:24:44,551 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5dcfd84eb369d011a0951d03c984f736 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-27T12:24:44,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(8855): Flush requested on 5dcfd84eb369d011a0951d03c984f736 2024-11-27T12:24:44,571 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411272715c88339fe4f3c8b8a2526d98f63ad_5dcfd84eb369d011a0951d03c984f736 is 50, key is test_row_0/A:col10/1732710284550/Put/seqid=0 2024-11-27T12:24:44,580 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073741947_1123 (size=14594) 2024-11-27T12:24:44,581 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:44,587 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411272715c88339fe4f3c8b8a2526d98f63ad_5dcfd84eb369d011a0951d03c984f736 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411272715c88339fe4f3c8b8a2526d98f63ad_5dcfd84eb369d011a0951d03c984f736 2024-11-27T12:24:44,592 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-27T12:24:44,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736. 2024-11-27T12:24:44,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736. as already flushing 2024-11-27T12:24:44,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736. 2024-11-27T12:24:44,593 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 5dcfd84eb369d011a0951d03c984f736, NAME => 'TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:24:44,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 5dcfd84eb369d011a0951d03c984f736, NAME => 'TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:24:44,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5dcfd84eb369d011a0951d03c984f736, NAME => 'TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5dcfd84eb369d011a0951d03c984f736, NAME => 'TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:24:44,594 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/.tmp/A/414c1c6e7eb14998800c17b8a2c34935, store: [table=TestAcidGuarantees family=A region=5dcfd84eb369d011a0951d03c984f736] 2024-11-27T12:24:44,595 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/.tmp/A/414c1c6e7eb14998800c17b8a2c34935 is 175, key is test_row_0/A:col10/1732710284550/Put/seqid=0 2024-11-27T12:24:44,608 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:44,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44010 deadline: 1732710344607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:44,609 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:44,609 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:44,609 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:24:44,614 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:44,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44028 deadline: 1732710344610, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:44,614 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073741948_1124 (size=39549) 2024-11-27T12:24:44,615 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:44,615 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:44,615 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:24:44,616 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=55, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/.tmp/A/414c1c6e7eb14998800c17b8a2c34935 2024-11-27T12:24:44,634 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/.tmp/B/0a7033c1fee349d688893b3bd4a6f02d is 50, key is test_row_0/B:col10/1732710284550/Put/seqid=0 2024-11-27T12:24:44,655 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073741949_1125 (size=12001) 2024-11-27T12:24:44,662 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:44,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44064 deadline: 1732710344658, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:44,663 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:44,663 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:44,663 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:24:44,670 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:44,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44012 deadline: 1732710344668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:44,670 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:44,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44052 deadline: 1732710344669, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:44,672 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:44,672 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:44,672 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:24:44,672 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:44,673 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:44,673 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:24:44,718 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:44,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44010 deadline: 1732710344718, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:44,719 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:44,719 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:44,719 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:44,719 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:24:44,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44028 deadline: 1732710344718, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:44,719 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:44,720 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:44,720 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:24:44,746 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-27T12:24:44,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736. 2024-11-27T12:24:44,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736. as already flushing 2024-11-27T12:24:44,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736. 2024-11-27T12:24:44,746 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 5dcfd84eb369d011a0951d03c984f736, NAME => 'TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:24:44,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 5dcfd84eb369d011a0951d03c984f736, NAME => 'TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:24:44,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-27T12:24:44,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5dcfd84eb369d011a0951d03c984f736, NAME => 'TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5dcfd84eb369d011a0951d03c984f736, NAME => 'TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:24:44,901 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-27T12:24:44,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736. 2024-11-27T12:24:44,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736. as already flushing 2024-11-27T12:24:44,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736. 2024-11-27T12:24:44,901 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 5dcfd84eb369d011a0951d03c984f736, NAME => 'TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:24:44,902 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 5dcfd84eb369d011a0951d03c984f736, NAME => 'TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:24:44,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5dcfd84eb369d011a0951d03c984f736, NAME => 'TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5dcfd84eb369d011a0951d03c984f736, NAME => 'TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:24:44,929 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-27T12:24:44,930 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:44,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44028 deadline: 1732710344928, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:44,931 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:44,931 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:44,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44010 deadline: 1732710344928, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:44,931 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:44,931 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:24:44,932 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:44,932 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:44,932 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:24:45,054 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-27T12:24:45,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736. 2024-11-27T12:24:45,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736. as already flushing 2024-11-27T12:24:45,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736. 2024-11-27T12:24:45,054 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 5dcfd84eb369d011a0951d03c984f736, NAME => 'TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:24:45,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 5dcfd84eb369d011a0951d03c984f736, NAME => 'TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:24:45,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5dcfd84eb369d011a0951d03c984f736, NAME => 'TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5dcfd84eb369d011a0951d03c984f736, NAME => 'TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:24:45,056 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=55 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/.tmp/B/0a7033c1fee349d688893b3bd4a6f02d 2024-11-27T12:24:45,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-27T12:24:45,065 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/.tmp/C/39b7bd0de2a94d18af17d35542f12ede is 50, key is test_row_0/C:col10/1732710284550/Put/seqid=0 2024-11-27T12:24:45,079 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073741950_1126 (size=12001) 2024-11-27T12:24:45,080 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=55 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/.tmp/C/39b7bd0de2a94d18af17d35542f12ede 2024-11-27T12:24:45,087 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/.tmp/A/414c1c6e7eb14998800c17b8a2c34935 as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/A/414c1c6e7eb14998800c17b8a2c34935 2024-11-27T12:24:45,092 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/A/414c1c6e7eb14998800c17b8a2c34935, entries=200, sequenceid=55, filesize=38.6 K 2024-11-27T12:24:45,094 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/.tmp/B/0a7033c1fee349d688893b3bd4a6f02d as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/B/0a7033c1fee349d688893b3bd4a6f02d 2024-11-27T12:24:45,099 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/B/0a7033c1fee349d688893b3bd4a6f02d, entries=150, sequenceid=55, filesize=11.7 K 2024-11-27T12:24:45,100 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/.tmp/C/39b7bd0de2a94d18af17d35542f12ede as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/C/39b7bd0de2a94d18af17d35542f12ede 2024-11-27T12:24:45,105 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/C/39b7bd0de2a94d18af17d35542f12ede, entries=150, sequenceid=55, filesize=11.7 K 2024-11-27T12:24:45,106 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 5dcfd84eb369d011a0951d03c984f736 in 556ms, sequenceid=55, compaction requested=true 2024-11-27T12:24:45,106 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5dcfd84eb369d011a0951d03c984f736: 2024-11-27T12:24:45,107 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5dcfd84eb369d011a0951d03c984f736:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T12:24:45,107 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T12:24:45,107 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T12:24:45,107 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5dcfd84eb369d011a0951d03c984f736:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T12:24:45,107 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T12:24:45,107 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T12:24:45,107 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5dcfd84eb369d011a0951d03c984f736:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T12:24:45,107 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T12:24:45,108 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110053 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T12:24:45,108 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HStore(1541): 5dcfd84eb369d011a0951d03c984f736/A is initiating minor compaction (all files) 2024-11-27T12:24:45,108 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5dcfd84eb369d011a0951d03c984f736/A in TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736. 2024-11-27T12:24:45,108 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/A/087bb11cf4cb49c1b6f5f185314181d9, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/A/f96b021a7716418ca7c9c432ef18b536, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/A/414c1c6e7eb14998800c17b8a2c34935] into tmpdir=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/.tmp, totalSize=107.5 K 2024-11-27T12:24:45,108 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736. 2024-11-27T12:24:45,108 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736. files: [hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/A/087bb11cf4cb49c1b6f5f185314181d9, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/A/f96b021a7716418ca7c9c432ef18b536, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/A/414c1c6e7eb14998800c17b8a2c34935] 2024-11-27T12:24:45,109 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T12:24:45,109 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HStore(1541): 5dcfd84eb369d011a0951d03c984f736/B is initiating minor compaction (all files) 2024-11-27T12:24:45,109 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5dcfd84eb369d011a0951d03c984f736/B in TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736. 2024-11-27T12:24:45,109 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/B/583ff06821a042adba3819a0e06cc3e4, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/B/c3cc05cc3520420491c4ce2d8926e957, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/B/0a7033c1fee349d688893b3bd4a6f02d] into tmpdir=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/.tmp, totalSize=35.2 K 2024-11-27T12:24:45,109 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] compactions.Compactor(225): Compacting 087bb11cf4cb49c1b6f5f185314181d9, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=17, earliestPutTs=1732710282283 2024-11-27T12:24:45,109 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] compactions.Compactor(225): Compacting f96b021a7716418ca7c9c432ef18b536, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=42, earliestPutTs=1732710282326 2024-11-27T12:24:45,109 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.Compactor(225): Compacting 583ff06821a042adba3819a0e06cc3e4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=17, earliestPutTs=1732710282284 2024-11-27T12:24:45,109 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] compactions.Compactor(225): Compacting 414c1c6e7eb14998800c17b8a2c34935, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=55, earliestPutTs=1732710283504 2024-11-27T12:24:45,110 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.Compactor(225): Compacting c3cc05cc3520420491c4ce2d8926e957, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=42, earliestPutTs=1732710282326 2024-11-27T12:24:45,110 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.Compactor(225): Compacting 0a7033c1fee349d688893b3bd4a6f02d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=55, earliestPutTs=1732710283504 2024-11-27T12:24:45,118 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=5dcfd84eb369d011a0951d03c984f736] 2024-11-27T12:24:45,122 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241127a6428212ce7e4051bd5339015d61bb14_5dcfd84eb369d011a0951d03c984f736 store=[table=TestAcidGuarantees family=A region=5dcfd84eb369d011a0951d03c984f736] 2024-11-27T12:24:45,123 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5dcfd84eb369d011a0951d03c984f736#B#compaction#109 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-27T12:24:45,123 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/.tmp/B/5b7cf336c8f84d09921b6fd62da7b25f is 50, key is test_row_0/B:col10/1732710284550/Put/seqid=0 2024-11-27T12:24:45,129 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241127a6428212ce7e4051bd5339015d61bb14_5dcfd84eb369d011a0951d03c984f736, store=[table=TestAcidGuarantees family=A region=5dcfd84eb369d011a0951d03c984f736] 2024-11-27T12:24:45,129 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127a6428212ce7e4051bd5339015d61bb14_5dcfd84eb369d011a0951d03c984f736 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=5dcfd84eb369d011a0951d03c984f736] 2024-11-27T12:24:45,150 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073741951_1127 (size=12104) 2024-11-27T12:24:45,160 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/.tmp/B/5b7cf336c8f84d09921b6fd62da7b25f as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/B/5b7cf336c8f84d09921b6fd62da7b25f 2024-11-27T12:24:45,171 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5dcfd84eb369d011a0951d03c984f736/B of 5dcfd84eb369d011a0951d03c984f736 into 5b7cf336c8f84d09921b6fd62da7b25f(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T12:24:45,172 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5dcfd84eb369d011a0951d03c984f736: 2024-11-27T12:24:45,172 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., storeName=5dcfd84eb369d011a0951d03c984f736/B, priority=13, startTime=1732710285107; duration=0sec 2024-11-27T12:24:45,172 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T12:24:45,172 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5dcfd84eb369d011a0951d03c984f736:B 2024-11-27T12:24:45,172 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T12:24:45,175 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T12:24:45,176 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HStore(1541): 5dcfd84eb369d011a0951d03c984f736/C is initiating minor compaction (all files) 2024-11-27T12:24:45,176 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5dcfd84eb369d011a0951d03c984f736/C in TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736. 2024-11-27T12:24:45,176 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/C/15779297379e4155b260778392f26a78, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/C/22e85f9daff94ca59a9ae29b0df314d9, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/C/39b7bd0de2a94d18af17d35542f12ede] into tmpdir=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/.tmp, totalSize=35.2 K 2024-11-27T12:24:45,176 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.Compactor(225): Compacting 15779297379e4155b260778392f26a78, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=17, earliestPutTs=1732710282284 2024-11-27T12:24:45,177 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.Compactor(225): Compacting 22e85f9daff94ca59a9ae29b0df314d9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=42, earliestPutTs=1732710282326 2024-11-27T12:24:45,177 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.Compactor(225): Compacting 39b7bd0de2a94d18af17d35542f12ede, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=55, earliestPutTs=1732710283504 2024-11-27T12:24:45,179 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073741952_1128 (size=4469) 2024-11-27T12:24:45,181 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5dcfd84eb369d011a0951d03c984f736#A#compaction#108 average throughput is 0.39 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T12:24:45,181 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/.tmp/A/46bb966630b2406ea1fd162932e6ffc3 is 175, key is test_row_0/A:col10/1732710284550/Put/seqid=0 2024-11-27T12:24:45,189 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073741953_1129 (size=31058) 2024-11-27T12:24:45,196 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/.tmp/A/46bb966630b2406ea1fd162932e6ffc3 as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/A/46bb966630b2406ea1fd162932e6ffc3 2024-11-27T12:24:45,199 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5dcfd84eb369d011a0951d03c984f736#C#compaction#110 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T12:24:45,200 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/.tmp/C/5cd61286223d4c38b7803121392d3c35 is 50, key is test_row_0/C:col10/1732710284550/Put/seqid=0 2024-11-27T12:24:45,203 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5dcfd84eb369d011a0951d03c984f736/A of 5dcfd84eb369d011a0951d03c984f736 into 46bb966630b2406ea1fd162932e6ffc3(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T12:24:45,203 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5dcfd84eb369d011a0951d03c984f736: 2024-11-27T12:24:45,203 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., storeName=5dcfd84eb369d011a0951d03c984f736/A, priority=13, startTime=1732710285107; duration=0sec 2024-11-27T12:24:45,203 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T12:24:45,203 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5dcfd84eb369d011a0951d03c984f736:A 2024-11-27T12:24:45,207 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-27T12:24:45,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736. 2024-11-27T12:24:45,208 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2902): Flushing 5dcfd84eb369d011a0951d03c984f736 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-27T12:24:45,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411273cb2605994f24c41a2f66534a5acedf7_5dcfd84eb369d011a0951d03c984f736 is 50, key is test_row_0/A:col10/1732710284593/Put/seqid=0 2024-11-27T12:24:45,229 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073741954_1130 (size=12104) 2024-11-27T12:24:45,240 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/.tmp/C/5cd61286223d4c38b7803121392d3c35 as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/C/5cd61286223d4c38b7803121392d3c35 2024-11-27T12:24:45,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(8855): Flush requested on 5dcfd84eb369d011a0951d03c984f736 2024-11-27T12:24:45,241 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736. as already flushing 2024-11-27T12:24:45,244 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073741955_1131 (size=12154) 2024-11-27T12:24:45,248 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5dcfd84eb369d011a0951d03c984f736/C of 5dcfd84eb369d011a0951d03c984f736 into 5cd61286223d4c38b7803121392d3c35(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T12:24:45,248 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5dcfd84eb369d011a0951d03c984f736: 2024-11-27T12:24:45,248 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., storeName=5dcfd84eb369d011a0951d03c984f736/C, priority=13, startTime=1732710285107; duration=0sec 2024-11-27T12:24:45,248 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T12:24:45,248 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5dcfd84eb369d011a0951d03c984f736:C 2024-11-27T12:24:45,260 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:45,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44010 deadline: 1732710345257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:45,261 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:45,261 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:45,261 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:24:45,265 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:45,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44028 deadline: 1732710345261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:45,265 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:45,265 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:45,265 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:24:45,368 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:45,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44010 deadline: 1732710345368, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:45,368 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:45,369 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:45,369 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:24:45,370 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:45,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44028 deadline: 1732710345368, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:45,371 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:45,371 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:45,371 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:24:45,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-27T12:24:45,583 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:45,583 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:45,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44028 deadline: 1732710345578, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:45,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44010 deadline: 1732710345578, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:45,583 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:45,583 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:45,583 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:45,584 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:45,584 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:24:45,584 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:24:45,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:45,649 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411273cb2605994f24c41a2f66534a5acedf7_5dcfd84eb369d011a0951d03c984f736 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411273cb2605994f24c41a2f66534a5acedf7_5dcfd84eb369d011a0951d03c984f736 2024-11-27T12:24:45,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/.tmp/A/db240c5207ea45f4bc89f2d332c1d35e, store: [table=TestAcidGuarantees family=A region=5dcfd84eb369d011a0951d03c984f736] 2024-11-27T12:24:45,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/.tmp/A/db240c5207ea45f4bc89f2d332c1d35e is 175, key is test_row_0/A:col10/1732710284593/Put/seqid=0 2024-11-27T12:24:45,654 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073741956_1132 (size=30955) 2024-11-27T12:24:45,670 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:45,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44064 deadline: 1732710345668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:45,670 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 1ba5af43fd99,36251,1732710241631 for 'test_row_0' in 5dcfd84eb369d011a0951d03c984f736 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2156 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T12:24:45,671 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:45,671 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:45,671 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:24:45,688 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:45,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44052 deadline: 1732710345688, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:45,689 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:45,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44012 deadline: 1732710345688, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:45,689 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 1ba5af43fd99,36251,1732710241631 for 'test_row_1' in 5dcfd84eb369d011a0951d03c984f736 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2172 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T12:24:45,689 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:45,689 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:45,689 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:24:45,689 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 1ba5af43fd99,36251,1732710241631 for 'test_row_0' in 5dcfd84eb369d011a0951d03c984f736 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2171 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T12:24:45,689 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:45,690 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:45,690 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:24:45,889 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:45,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44010 deadline: 1732710345888, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:45,889 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:45,890 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:45,890 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:24:45,890 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:45,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44028 deadline: 1732710345888, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:45,891 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:45,891 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:45,891 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:24:46,057 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=80, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/.tmp/A/db240c5207ea45f4bc89f2d332c1d35e 2024-11-27T12:24:46,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/.tmp/B/0555cd7548fe4505a89fd66d47434099 is 50, key is test_row_0/B:col10/1732710284593/Put/seqid=0 2024-11-27T12:24:46,072 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073741957_1133 (size=12001) 2024-11-27T12:24:46,284 INFO [master/1ba5af43fd99:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-11-27T12:24:46,285 INFO [master/1ba5af43fd99:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-11-27T12:24:46,398 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:46,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44010 deadline: 1732710346398, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:46,399 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:46,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44028 deadline: 1732710346398, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:46,399 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:46,399 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:46,399 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:24:46,399 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:46,399 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:46,399 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:24:46,472 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=80 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/.tmp/B/0555cd7548fe4505a89fd66d47434099 2024-11-27T12:24:46,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/.tmp/C/ddf504151c0f44f4ad64747ce1981b5d is 50, key is test_row_0/C:col10/1732710284593/Put/seqid=0 2024-11-27T12:24:46,510 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073741958_1134 (size=12001) 2024-11-27T12:24:46,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-27T12:24:46,910 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=80 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/.tmp/C/ddf504151c0f44f4ad64747ce1981b5d 2024-11-27T12:24:46,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/.tmp/A/db240c5207ea45f4bc89f2d332c1d35e as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/A/db240c5207ea45f4bc89f2d332c1d35e 2024-11-27T12:24:46,923 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/A/db240c5207ea45f4bc89f2d332c1d35e, entries=150, sequenceid=80, filesize=30.2 K 2024-11-27T12:24:46,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/.tmp/B/0555cd7548fe4505a89fd66d47434099 as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/B/0555cd7548fe4505a89fd66d47434099 2024-11-27T12:24:46,930 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/B/0555cd7548fe4505a89fd66d47434099, entries=150, sequenceid=80, filesize=11.7 K 2024-11-27T12:24:46,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/.tmp/C/ddf504151c0f44f4ad64747ce1981b5d as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/C/ddf504151c0f44f4ad64747ce1981b5d 2024-11-27T12:24:46,938 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/C/ddf504151c0f44f4ad64747ce1981b5d, entries=150, sequenceid=80, filesize=11.7 K 2024-11-27T12:24:46,939 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for 5dcfd84eb369d011a0951d03c984f736 in 1732ms, sequenceid=80, compaction requested=false 2024-11-27T12:24:46,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2603): Flush status journal for 5dcfd84eb369d011a0951d03c984f736: 2024-11-27T12:24:46,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736. 2024-11-27T12:24:46,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=43 2024-11-27T12:24:46,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.HMaster(4169): Remote procedure done, pid=43 2024-11-27T12:24:46,944 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=43, resume processing ppid=42 2024-11-27T12:24:46,944 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=43, ppid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.5080 sec 2024-11-27T12:24:46,947 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees in 2.5160 sec 2024-11-27T12:24:47,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(8855): Flush requested on 5dcfd84eb369d011a0951d03c984f736 2024-11-27T12:24:47,411 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T12:24:47,411 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5dcfd84eb369d011a0951d03c984f736 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-27T12:24:47,463 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411279478eb62e8bd46a2a95c36d4895ed1bc_5dcfd84eb369d011a0951d03c984f736 is 50, key is test_row_0/A:col10/1732710287410/Put/seqid=0 2024-11-27T12:24:47,488 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:47,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44028 deadline: 1732710347486, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:47,488 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:47,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44010 deadline: 1732710347487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:47,489 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:47,489 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:47,489 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:24:47,489 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:47,489 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:47,489 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:24:47,506 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073741959_1135 (size=12154) 2024-11-27T12:24:47,598 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:47,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44028 deadline: 1732710347598, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:47,599 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:47,599 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:47,599 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:47,599 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:24:47,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44010 deadline: 1732710347598, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:47,601 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:47,601 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:47,601 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:24:47,679 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:47,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44064 deadline: 1732710347678, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:47,680 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 1ba5af43fd99,36251,1732710241631 for 'test_row_0' in 5dcfd84eb369d011a0951d03c984f736 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4166 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T12:24:47,681 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:47,681 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:47,681 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:24:47,699 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:47,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44012 deadline: 1732710347698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:47,700 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 1ba5af43fd99,36251,1732710241631 for 'test_row_0' in 5dcfd84eb369d011a0951d03c984f736 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4182 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T12:24:47,700 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:47,700 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:47,700 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:24:47,710 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:47,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44052 deadline: 1732710347708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:47,711 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 1ba5af43fd99,36251,1732710241631 for 'test_row_1' in 5dcfd84eb369d011a0951d03c984f736 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4195 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T12:24:47,712 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:47,712 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:47,712 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:24:47,810 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:47,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44028 deadline: 1732710347808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:47,811 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:47,811 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:47,811 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:47,811 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:24:47,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44010 deadline: 1732710347808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:47,811 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:47,811 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:47,811 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:24:47,904 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:47,909 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411279478eb62e8bd46a2a95c36d4895ed1bc_5dcfd84eb369d011a0951d03c984f736 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411279478eb62e8bd46a2a95c36d4895ed1bc_5dcfd84eb369d011a0951d03c984f736 2024-11-27T12:24:47,910 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/.tmp/A/3b6cbf421a1b4d08abf085c06e11befc, store: [table=TestAcidGuarantees family=A region=5dcfd84eb369d011a0951d03c984f736] 2024-11-27T12:24:47,910 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/.tmp/A/3b6cbf421a1b4d08abf085c06e11befc is 175, key is test_row_0/A:col10/1732710287410/Put/seqid=0 2024-11-27T12:24:47,916 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073741960_1136 (size=30955) 2024-11-27T12:24:48,119 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:48,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44028 deadline: 1732710348119, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:48,120 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:48,120 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:48,120 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:24:48,123 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:48,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44010 deadline: 1732710348119, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:48,124 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:48,124 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:48,124 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:24:48,315 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=95, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/.tmp/A/3b6cbf421a1b4d08abf085c06e11befc 2024-11-27T12:24:48,324 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/.tmp/B/b14ac58ab7454b569b4e1c12c06918bf is 50, key is test_row_0/B:col10/1732710287410/Put/seqid=0 2024-11-27T12:24:48,328 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073741961_1137 (size=12001) 2024-11-27T12:24:48,329 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=95 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/.tmp/B/b14ac58ab7454b569b4e1c12c06918bf 2024-11-27T12:24:48,339 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/.tmp/C/003f0c162c9b4dc4b204b106a34c8c90 is 50, key is test_row_0/C:col10/1732710287410/Put/seqid=0 2024-11-27T12:24:48,343 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073741962_1138 (size=12001) 2024-11-27T12:24:48,344 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=95 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/.tmp/C/003f0c162c9b4dc4b204b106a34c8c90 2024-11-27T12:24:48,349 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/.tmp/A/3b6cbf421a1b4d08abf085c06e11befc as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/A/3b6cbf421a1b4d08abf085c06e11befc 2024-11-27T12:24:48,354 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/A/3b6cbf421a1b4d08abf085c06e11befc, entries=150, sequenceid=95, filesize=30.2 K 2024-11-27T12:24:48,355 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/.tmp/B/b14ac58ab7454b569b4e1c12c06918bf as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/B/b14ac58ab7454b569b4e1c12c06918bf 2024-11-27T12:24:48,360 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/B/b14ac58ab7454b569b4e1c12c06918bf, entries=150, sequenceid=95, filesize=11.7 K 2024-11-27T12:24:48,362 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/.tmp/C/003f0c162c9b4dc4b204b106a34c8c90 as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/C/003f0c162c9b4dc4b204b106a34c8c90 2024-11-27T12:24:48,368 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/C/003f0c162c9b4dc4b204b106a34c8c90, entries=150, sequenceid=95, filesize=11.7 K 2024-11-27T12:24:48,369 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=134.18 KB/137400 for 5dcfd84eb369d011a0951d03c984f736 in 958ms, sequenceid=95, compaction requested=true 2024-11-27T12:24:48,369 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5dcfd84eb369d011a0951d03c984f736: 2024-11-27T12:24:48,369 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5dcfd84eb369d011a0951d03c984f736:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T12:24:48,369 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T12:24:48,369 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T12:24:48,369 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5dcfd84eb369d011a0951d03c984f736:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T12:24:48,369 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T12:24:48,369 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T12:24:48,369 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5dcfd84eb369d011a0951d03c984f736:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T12:24:48,369 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T12:24:48,371 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 92968 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T12:24:48,371 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T12:24:48,371 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HStore(1541): 5dcfd84eb369d011a0951d03c984f736/B is initiating minor compaction (all files) 2024-11-27T12:24:48,371 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HStore(1541): 5dcfd84eb369d011a0951d03c984f736/A is initiating minor compaction (all files) 2024-11-27T12:24:48,371 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5dcfd84eb369d011a0951d03c984f736/B in TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736. 2024-11-27T12:24:48,371 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5dcfd84eb369d011a0951d03c984f736/A in TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736. 2024-11-27T12:24:48,371 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/A/46bb966630b2406ea1fd162932e6ffc3, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/A/db240c5207ea45f4bc89f2d332c1d35e, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/A/3b6cbf421a1b4d08abf085c06e11befc] into tmpdir=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/.tmp, totalSize=90.8 K 2024-11-27T12:24:48,371 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/B/5b7cf336c8f84d09921b6fd62da7b25f, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/B/0555cd7548fe4505a89fd66d47434099, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/B/b14ac58ab7454b569b4e1c12c06918bf] into tmpdir=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/.tmp, totalSize=35.3 K 2024-11-27T12:24:48,371 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736. 2024-11-27T12:24:48,371 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736. files: [hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/A/46bb966630b2406ea1fd162932e6ffc3, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/A/db240c5207ea45f4bc89f2d332c1d35e, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/A/3b6cbf421a1b4d08abf085c06e11befc] 2024-11-27T12:24:48,371 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.Compactor(225): Compacting 5b7cf336c8f84d09921b6fd62da7b25f, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=55, earliestPutTs=1732710283504 2024-11-27T12:24:48,372 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] compactions.Compactor(225): Compacting 46bb966630b2406ea1fd162932e6ffc3, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=55, earliestPutTs=1732710283504 2024-11-27T12:24:48,372 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.Compactor(225): Compacting 0555cd7548fe4505a89fd66d47434099, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=80, earliestPutTs=1732710284593 2024-11-27T12:24:48,372 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] compactions.Compactor(225): Compacting db240c5207ea45f4bc89f2d332c1d35e, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=80, earliestPutTs=1732710284593 2024-11-27T12:24:48,373 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.Compactor(225): Compacting b14ac58ab7454b569b4e1c12c06918bf, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1732710285256 2024-11-27T12:24:48,373 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3b6cbf421a1b4d08abf085c06e11befc, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1732710285256 2024-11-27T12:24:48,384 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5dcfd84eb369d011a0951d03c984f736#B#compaction#117 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T12:24:48,384 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/.tmp/B/722bb37ed1bb491bb719a577c007ff73 is 50, key is test_row_0/B:col10/1732710287410/Put/seqid=0 2024-11-27T12:24:48,385 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=5dcfd84eb369d011a0951d03c984f736] 2024-11-27T12:24:48,388 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024112793186927e86543e8b23b2bdf9f2b9c30_5dcfd84eb369d011a0951d03c984f736 store=[table=TestAcidGuarantees family=A region=5dcfd84eb369d011a0951d03c984f736] 2024-11-27T12:24:48,392 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024112793186927e86543e8b23b2bdf9f2b9c30_5dcfd84eb369d011a0951d03c984f736, store=[table=TestAcidGuarantees family=A region=5dcfd84eb369d011a0951d03c984f736] 2024-11-27T12:24:48,392 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112793186927e86543e8b23b2bdf9f2b9c30_5dcfd84eb369d011a0951d03c984f736 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=5dcfd84eb369d011a0951d03c984f736] 2024-11-27T12:24:48,397 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073741963_1139 (size=12207) 2024-11-27T12:24:48,405 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/.tmp/B/722bb37ed1bb491bb719a577c007ff73 as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/B/722bb37ed1bb491bb719a577c007ff73 2024-11-27T12:24:48,406 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073741964_1140 (size=4469) 2024-11-27T12:24:48,408 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5dcfd84eb369d011a0951d03c984f736#A#compaction#118 average throughput is 1.11 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T12:24:48,409 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/.tmp/A/2d76e012974b4244a2e0c7b36db4bda5 is 175, key is test_row_0/A:col10/1732710287410/Put/seqid=0 2024-11-27T12:24:48,413 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5dcfd84eb369d011a0951d03c984f736/B of 5dcfd84eb369d011a0951d03c984f736 into 722bb37ed1bb491bb719a577c007ff73(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T12:24:48,414 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5dcfd84eb369d011a0951d03c984f736: 2024-11-27T12:24:48,414 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., storeName=5dcfd84eb369d011a0951d03c984f736/B, priority=13, startTime=1732710288369; duration=0sec 2024-11-27T12:24:48,414 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T12:24:48,414 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5dcfd84eb369d011a0951d03c984f736:B 2024-11-27T12:24:48,414 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T12:24:48,415 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T12:24:48,415 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HStore(1541): 5dcfd84eb369d011a0951d03c984f736/C is initiating minor compaction (all files) 2024-11-27T12:24:48,416 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5dcfd84eb369d011a0951d03c984f736/C in TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736. 2024-11-27T12:24:48,416 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073741965_1141 (size=31161) 2024-11-27T12:24:48,417 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/C/5cd61286223d4c38b7803121392d3c35, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/C/ddf504151c0f44f4ad64747ce1981b5d, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/C/003f0c162c9b4dc4b204b106a34c8c90] into tmpdir=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/.tmp, totalSize=35.3 K 2024-11-27T12:24:48,418 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.Compactor(225): Compacting 5cd61286223d4c38b7803121392d3c35, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=55, earliestPutTs=1732710283504 2024-11-27T12:24:48,418 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.Compactor(225): Compacting ddf504151c0f44f4ad64747ce1981b5d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=80, earliestPutTs=1732710284593 2024-11-27T12:24:48,419 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.Compactor(225): Compacting 003f0c162c9b4dc4b204b106a34c8c90, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1732710285256 2024-11-27T12:24:48,423 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/.tmp/A/2d76e012974b4244a2e0c7b36db4bda5 as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/A/2d76e012974b4244a2e0c7b36db4bda5 2024-11-27T12:24:48,429 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5dcfd84eb369d011a0951d03c984f736/A of 5dcfd84eb369d011a0951d03c984f736 into 2d76e012974b4244a2e0c7b36db4bda5(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T12:24:48,429 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5dcfd84eb369d011a0951d03c984f736: 2024-11-27T12:24:48,429 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., storeName=5dcfd84eb369d011a0951d03c984f736/A, priority=13, startTime=1732710288369; duration=0sec 2024-11-27T12:24:48,429 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T12:24:48,429 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5dcfd84eb369d011a0951d03c984f736:A 2024-11-27T12:24:48,430 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5dcfd84eb369d011a0951d03c984f736#C#compaction#119 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T12:24:48,430 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/.tmp/C/c72219f77d354bc1b75f4ebde7cff5bb is 50, key is test_row_0/C:col10/1732710287410/Put/seqid=0 2024-11-27T12:24:48,445 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073741966_1142 (size=12207) 2024-11-27T12:24:48,454 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/.tmp/C/c72219f77d354bc1b75f4ebde7cff5bb as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/C/c72219f77d354bc1b75f4ebde7cff5bb 2024-11-27T12:24:48,460 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5dcfd84eb369d011a0951d03c984f736/C of 5dcfd84eb369d011a0951d03c984f736 into c72219f77d354bc1b75f4ebde7cff5bb(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T12:24:48,460 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5dcfd84eb369d011a0951d03c984f736: 2024-11-27T12:24:48,460 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., storeName=5dcfd84eb369d011a0951d03c984f736/C, priority=13, startTime=1732710288369; duration=0sec 2024-11-27T12:24:48,460 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T12:24:48,460 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5dcfd84eb369d011a0951d03c984f736:C 2024-11-27T12:24:48,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-27T12:24:48,599 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T12:24:48,601 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-27T12:24:48,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] procedure2.ProcedureExecutor(1139): Stored pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees 2024-11-27T12:24:48,603 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T12:24:48,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-27T12:24:48,604 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T12:24:48,604 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=45, ppid=44, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T12:24:48,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(8855): Flush requested on 5dcfd84eb369d011a0951d03c984f736 2024-11-27T12:24:48,631 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T12:24:48,631 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5dcfd84eb369d011a0951d03c984f736 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-27T12:24:48,646 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411277e6cb2181a244bf0bd5081cca9d3dda1_5dcfd84eb369d011a0951d03c984f736 is 50, key is test_row_0/A:col10/1732710287459/Put/seqid=0 2024-11-27T12:24:48,658 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:48,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44028 deadline: 1732710348655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:48,659 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:48,659 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:48,659 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:24:48,660 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:48,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44010 deadline: 1732710348658, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:48,661 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:48,661 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:48,661 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:24:48,671 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073741967_1143 (size=14594) 2024-11-27T12:24:48,672 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:48,679 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411277e6cb2181a244bf0bd5081cca9d3dda1_5dcfd84eb369d011a0951d03c984f736 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411277e6cb2181a244bf0bd5081cca9d3dda1_5dcfd84eb369d011a0951d03c984f736 2024-11-27T12:24:48,680 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/.tmp/A/804bb279bfac454c9d73ce492a45e560, store: [table=TestAcidGuarantees family=A region=5dcfd84eb369d011a0951d03c984f736] 2024-11-27T12:24:48,681 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/.tmp/A/804bb279bfac454c9d73ce492a45e560 is 175, key is test_row_0/A:col10/1732710287459/Put/seqid=0 2024-11-27T12:24:48,701 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073741968_1144 (size=39549) 2024-11-27T12:24:48,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-27T12:24:48,758 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-27T12:24:48,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736. 2024-11-27T12:24:48,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736. as already flushing 2024-11-27T12:24:48,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736. 2024-11-27T12:24:48,759 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 5dcfd84eb369d011a0951d03c984f736, NAME => 'TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:24:48,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 5dcfd84eb369d011a0951d03c984f736, NAME => 'TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:24:48,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5dcfd84eb369d011a0951d03c984f736, NAME => 'TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5dcfd84eb369d011a0951d03c984f736, NAME => 'TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:24:48,768 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:48,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44010 deadline: 1732710348768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:48,769 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:48,769 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:48,769 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:48,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44028 deadline: 1732710348768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:48,769 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:24:48,771 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:48,771 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:48,771 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:24:48,913 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-27T12:24:48,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736. 2024-11-27T12:24:48,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736. as already flushing 2024-11-27T12:24:48,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736. 2024-11-27T12:24:48,913 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 5dcfd84eb369d011a0951d03c984f736, NAME => 'TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:24:48,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 5dcfd84eb369d011a0951d03c984f736, NAME => 'TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:24:48,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5dcfd84eb369d011a0951d03c984f736, NAME => 'TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5dcfd84eb369d011a0951d03c984f736, NAME => 'TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:24:48,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-27T12:24:48,979 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:48,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44010 deadline: 1732710348978, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:48,980 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:48,980 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:48,980 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:48,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44028 deadline: 1732710348978, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:48,980 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:24:48,980 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:48,980 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:48,981 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:24:49,066 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-27T12:24:49,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736. 2024-11-27T12:24:49,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736. as already flushing 2024-11-27T12:24:49,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736. 2024-11-27T12:24:49,066 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 5dcfd84eb369d011a0951d03c984f736, NAME => 'TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:24:49,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 5dcfd84eb369d011a0951d03c984f736, NAME => 'TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:24:49,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5dcfd84eb369d011a0951d03c984f736, NAME => 'TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5dcfd84eb369d011a0951d03c984f736, NAME => 'TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:24:49,105 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=122, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/.tmp/A/804bb279bfac454c9d73ce492a45e560 2024-11-27T12:24:49,117 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/.tmp/B/d0eebbcdf107487e9c5d5839048de3f0 is 50, key is test_row_0/B:col10/1732710287459/Put/seqid=0 2024-11-27T12:24:49,120 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073741969_1145 (size=12001) 2024-11-27T12:24:49,129 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=122 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/.tmp/B/d0eebbcdf107487e9c5d5839048de3f0 2024-11-27T12:24:49,139 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/.tmp/C/53c96489235d4ef1b96340d3b844a0d3 is 50, key is test_row_0/C:col10/1732710287459/Put/seqid=0 2024-11-27T12:24:49,149 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073741970_1146 (size=12001) 2024-11-27T12:24:49,219 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-27T12:24:49,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736. 2024-11-27T12:24:49,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736. as already flushing 2024-11-27T12:24:49,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736. 2024-11-27T12:24:49,219 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 5dcfd84eb369d011a0951d03c984f736, NAME => 'TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:24:49,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 5dcfd84eb369d011a0951d03c984f736, NAME => 'TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:24:49,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5dcfd84eb369d011a0951d03c984f736, NAME => 'TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5dcfd84eb369d011a0951d03c984f736, NAME => 'TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:24:49,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-27T12:24:49,291 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:49,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44028 deadline: 1732710349288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:49,292 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:49,291 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:49,292 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:49,292 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:24:49,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44010 deadline: 1732710349288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:49,292 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:49,292 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:49,292 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:24:49,373 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-27T12:24:49,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736. 2024-11-27T12:24:49,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736. as already flushing 2024-11-27T12:24:49,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736. 2024-11-27T12:24:49,373 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 5dcfd84eb369d011a0951d03c984f736, NAME => 'TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:24:49,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 5dcfd84eb369d011a0951d03c984f736, NAME => 'TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:24:49,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5dcfd84eb369d011a0951d03c984f736, NAME => 'TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5dcfd84eb369d011a0951d03c984f736, NAME => 'TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:24:49,526 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-27T12:24:49,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736. 2024-11-27T12:24:49,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736. as already flushing 2024-11-27T12:24:49,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736. 2024-11-27T12:24:49,526 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 5dcfd84eb369d011a0951d03c984f736, NAME => 'TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:24:49,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 5dcfd84eb369d011a0951d03c984f736, NAME => 'TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:24:49,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5dcfd84eb369d011a0951d03c984f736, NAME => 'TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5dcfd84eb369d011a0951d03c984f736, NAME => 'TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:24:49,550 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=122 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/.tmp/C/53c96489235d4ef1b96340d3b844a0d3 2024-11-27T12:24:49,557 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/.tmp/A/804bb279bfac454c9d73ce492a45e560 as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/A/804bb279bfac454c9d73ce492a45e560 2024-11-27T12:24:49,563 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/A/804bb279bfac454c9d73ce492a45e560, entries=200, sequenceid=122, filesize=38.6 K 2024-11-27T12:24:49,564 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/.tmp/B/d0eebbcdf107487e9c5d5839048de3f0 as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/B/d0eebbcdf107487e9c5d5839048de3f0 2024-11-27T12:24:49,572 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/B/d0eebbcdf107487e9c5d5839048de3f0, entries=150, sequenceid=122, filesize=11.7 K 2024-11-27T12:24:49,573 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/.tmp/C/53c96489235d4ef1b96340d3b844a0d3 as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/C/53c96489235d4ef1b96340d3b844a0d3 2024-11-27T12:24:49,580 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/C/53c96489235d4ef1b96340d3b844a0d3, entries=150, sequenceid=122, filesize=11.7 K 2024-11-27T12:24:49,581 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 5dcfd84eb369d011a0951d03c984f736 in 950ms, sequenceid=122, compaction requested=false 2024-11-27T12:24:49,581 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5dcfd84eb369d011a0951d03c984f736: 2024-11-27T12:24:49,679 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-27T12:24:49,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736. 2024-11-27T12:24:49,679 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2902): Flushing 5dcfd84eb369d011a0951d03c984f736 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-27T12:24:49,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411274cdfccf735de4d50a4bda8ac37409551_5dcfd84eb369d011a0951d03c984f736 is 50, key is test_row_0/A:col10/1732710288645/Put/seqid=0 2024-11-27T12:24:49,707 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073741971_1147 (size=12304) 2024-11-27T12:24:49,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:49,717 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411274cdfccf735de4d50a4bda8ac37409551_5dcfd84eb369d011a0951d03c984f736 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411274cdfccf735de4d50a4bda8ac37409551_5dcfd84eb369d011a0951d03c984f736 2024-11-27T12:24:49,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/.tmp/A/4343d343fba043e3b67f5f19dd5f4fbb, store: [table=TestAcidGuarantees family=A region=5dcfd84eb369d011a0951d03c984f736] 2024-11-27T12:24:49,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/.tmp/A/4343d343fba043e3b67f5f19dd5f4fbb is 175, key is test_row_0/A:col10/1732710288645/Put/seqid=0 2024-11-27T12:24:49,732 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073741972_1148 (size=31105) 2024-11-27T12:24:49,734 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=134, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/.tmp/A/4343d343fba043e3b67f5f19dd5f4fbb 2024-11-27T12:24:49,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-27T12:24:49,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/.tmp/B/ef23e15caac742c896111a802b9214d4 is 50, key is test_row_0/B:col10/1732710288645/Put/seqid=0 2024-11-27T12:24:49,766 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073741973_1149 (size=12151) 2024-11-27T12:24:49,768 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=134 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/.tmp/B/ef23e15caac742c896111a802b9214d4 2024-11-27T12:24:49,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/.tmp/C/5034c2ba00694a2fb3caa87aa991bac5 is 50, key is test_row_0/C:col10/1732710288645/Put/seqid=0 2024-11-27T12:24:49,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(8855): Flush requested on 5dcfd84eb369d011a0951d03c984f736 2024-11-27T12:24:49,799 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736. as already flushing 2024-11-27T12:24:49,801 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073741974_1150 (size=12151) 2024-11-27T12:24:49,802 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=134 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/.tmp/C/5034c2ba00694a2fb3caa87aa991bac5 2024-11-27T12:24:49,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/.tmp/A/4343d343fba043e3b67f5f19dd5f4fbb as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/A/4343d343fba043e3b67f5f19dd5f4fbb 2024-11-27T12:24:49,814 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/A/4343d343fba043e3b67f5f19dd5f4fbb, entries=150, sequenceid=134, filesize=30.4 K 2024-11-27T12:24:49,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/.tmp/B/ef23e15caac742c896111a802b9214d4 as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/B/ef23e15caac742c896111a802b9214d4 2024-11-27T12:24:49,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:49,821 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/B/ef23e15caac742c896111a802b9214d4, entries=150, sequenceid=134, filesize=11.9 K 2024-11-27T12:24:49,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/.tmp/C/5034c2ba00694a2fb3caa87aa991bac5 as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/C/5034c2ba00694a2fb3caa87aa991bac5 2024-11-27T12:24:49,827 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/C/5034c2ba00694a2fb3caa87aa991bac5, entries=150, sequenceid=134, filesize=11.9 K 2024-11-27T12:24:49,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:49,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:49,832 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=20.13 KB/20610 for 5dcfd84eb369d011a0951d03c984f736 in 152ms, sequenceid=134, compaction requested=true 2024-11-27T12:24:49,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2603): Flush status journal for 5dcfd84eb369d011a0951d03c984f736: 2024-11-27T12:24:49,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736. 2024-11-27T12:24:49,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=45 2024-11-27T12:24:49,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.HMaster(4169): Remote procedure done, pid=45 2024-11-27T12:24:49,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:49,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:49,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:49,836 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=45, resume processing ppid=44 2024-11-27T12:24:49,836 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=45, ppid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2300 sec 2024-11-27T12:24:49,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:49,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:49,838 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees in 1.2350 sec 2024-11-27T12:24:49,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:49,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:49,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:49,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:49,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:49,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:49,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:49,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:49,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:49,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:49,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:49,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:49,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:49,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:49,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:49,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:49,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:49,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:49,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:49,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:49,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:49,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:49,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:49,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:49,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:49,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:49,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:49,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:49,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:49,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:49,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:49,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:49,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:49,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:49,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:49,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:49,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:49,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:49,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:49,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:49,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:49,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:49,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:49,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:49,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:49,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:49,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:49,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:49,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:49,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:49,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:49,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:49,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:49,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:49,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:49,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:49,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:49,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:49,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:49,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:49,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:49,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:49,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:49,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:49,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:49,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:49,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:49,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:49,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:49,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:49,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:49,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:49,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:49,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:49,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:49,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:49,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:49,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:49,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:49,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:49,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:49,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:49,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:49,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:49,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:49,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:49,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:49,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:49,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:49,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:49,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:49,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:49,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:49,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:49,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:49,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:49,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:49,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:49,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:49,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:49,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:49,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:49,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:49,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:49,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:49,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:49,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:49,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:49,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:49,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:49,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:49,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:49,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:49,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:49,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:49,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:49,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:49,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:49,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:49,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:49,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:49,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:49,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:49,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:49,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:49,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:49,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:49,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:49,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:49,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:49,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:49,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:49,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:49,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:49,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:49,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:49,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:49,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:49,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:49,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:49,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:49,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:49,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:49,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:49,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:49,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:49,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:49,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:49,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:49,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:49,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:49,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:49,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:49,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:49,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:49,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:49,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:49,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:49,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:49,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:49,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:49,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:49,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:49,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:49,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:49,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:49,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:49,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:49,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:49,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:49,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:49,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:49,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:49,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:49,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:49,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:49,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:49,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:49,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:49,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:49,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:49,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:49,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:49,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:49,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:49,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:49,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:49,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:49,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:49,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:49,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:49,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:49,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:49,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:49,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:49,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:49,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:49,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:49,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:49,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:49,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:49,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:49,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:49,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:49,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:49,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:49,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:49,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:49,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:49,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:49,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:49,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:49,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:49,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:49,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:49,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:49,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:49,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:49,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:49,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:49,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:49,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:49,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:49,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:49,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:49,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:49,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:49,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:49,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:49,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:49,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:49,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:49,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:49,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:49,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:49,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:49,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:49,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:49,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:49,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:49,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:49,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:49,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:49,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:49,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:49,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:49,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:49,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:49,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:49,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:49,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:49,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:49,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:49,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:49,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:49,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:49,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:49,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:49,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:49,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:49,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:49,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:49,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:49,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:49,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:49,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:49,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:49,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:49,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:49,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:49,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:49,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:49,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:49,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:49,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:49,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:49,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:49,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:49,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(8855): Flush requested on 5dcfd84eb369d011a0951d03c984f736 2024-11-27T12:24:49,943 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T12:24:49,943 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5dcfd84eb369d011a0951d03c984f736 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-27T12:24:49,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:49,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:49,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:49,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:49,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:49,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:49,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:49,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:49,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:49,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:49,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:49,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:49,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:49,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:49,956 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411275db2e51773a6487db7cf3339316a061b_5dcfd84eb369d011a0951d03c984f736 is 50, key is test_row_0/A:col10/1732710289905/Put/seqid=0 2024-11-27T12:24:49,979 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073741975_1151 (size=12304) 2024-11-27T12:24:49,980 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:49,984 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411275db2e51773a6487db7cf3339316a061b_5dcfd84eb369d011a0951d03c984f736 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411275db2e51773a6487db7cf3339316a061b_5dcfd84eb369d011a0951d03c984f736 2024-11-27T12:24:49,985 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/.tmp/A/262b8f0fb1384bb19c60a861b52968d6, store: [table=TestAcidGuarantees family=A region=5dcfd84eb369d011a0951d03c984f736] 2024-11-27T12:24:49,985 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/.tmp/A/262b8f0fb1384bb19c60a861b52968d6 is 175, key is test_row_0/A:col10/1732710289905/Put/seqid=0 2024-11-27T12:24:50,015 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073741976_1152 (size=31105) 2024-11-27T12:24:50,017 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=145, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/.tmp/A/262b8f0fb1384bb19c60a861b52968d6 2024-11-27T12:24:50,031 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/.tmp/B/1effae4977224fea96e128ba90c2c146 is 50, key is test_row_0/B:col10/1732710289905/Put/seqid=0 2024-11-27T12:24:50,069 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073741977_1153 (size=12151) 2024-11-27T12:24:50,071 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=145 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/.tmp/B/1effae4977224fea96e128ba90c2c146 2024-11-27T12:24:50,082 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/.tmp/C/1a7819d8f9424fe38ab206b97a858095 is 50, key is test_row_0/C:col10/1732710289905/Put/seqid=0 2024-11-27T12:24:50,093 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:50,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44028 deadline: 1732710350083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:50,093 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:50,093 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:50,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44010 deadline: 1732710350085, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:50,093 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:50,094 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:24:50,094 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:50,094 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:50,094 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:24:50,114 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073741978_1154 (size=12151) 2024-11-27T12:24:50,199 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:50,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44028 deadline: 1732710350198, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:50,199 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:50,200 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:50,200 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:50,200 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:24:50,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44010 deadline: 1732710350198, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:50,200 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:50,200 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:50,200 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:24:50,409 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:50,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44028 deadline: 1732710350407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:50,410 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:50,410 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:50,410 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:50,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44010 deadline: 1732710350408, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:50,410 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:24:50,410 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:50,410 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:50,410 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:24:50,514 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=145 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/.tmp/C/1a7819d8f9424fe38ab206b97a858095 2024-11-27T12:24:50,522 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/.tmp/A/262b8f0fb1384bb19c60a861b52968d6 as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/A/262b8f0fb1384bb19c60a861b52968d6 2024-11-27T12:24:50,532 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/A/262b8f0fb1384bb19c60a861b52968d6, entries=150, sequenceid=145, filesize=30.4 K 2024-11-27T12:24:50,534 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/.tmp/B/1effae4977224fea96e128ba90c2c146 as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/B/1effae4977224fea96e128ba90c2c146 2024-11-27T12:24:50,540 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/B/1effae4977224fea96e128ba90c2c146, entries=150, sequenceid=145, filesize=11.9 K 2024-11-27T12:24:50,543 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/.tmp/C/1a7819d8f9424fe38ab206b97a858095 as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/C/1a7819d8f9424fe38ab206b97a858095 2024-11-27T12:24:50,557 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/C/1a7819d8f9424fe38ab206b97a858095, entries=150, sequenceid=145, filesize=11.9 K 2024-11-27T12:24:50,558 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 5dcfd84eb369d011a0951d03c984f736 in 615ms, sequenceid=145, compaction requested=true 2024-11-27T12:24:50,558 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5dcfd84eb369d011a0951d03c984f736: 2024-11-27T12:24:50,559 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5dcfd84eb369d011a0951d03c984f736:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T12:24:50,559 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-27T12:24:50,559 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T12:24:50,559 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-27T12:24:50,559 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5dcfd84eb369d011a0951d03c984f736:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T12:24:50,559 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T12:24:50,559 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5dcfd84eb369d011a0951d03c984f736:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T12:24:50,559 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T12:24:50,561 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48510 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-27T12:24:50,561 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HStore(1541): 5dcfd84eb369d011a0951d03c984f736/B is initiating minor compaction (all files) 2024-11-27T12:24:50,561 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5dcfd84eb369d011a0951d03c984f736/B in TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736. 2024-11-27T12:24:50,562 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/B/722bb37ed1bb491bb719a577c007ff73, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/B/d0eebbcdf107487e9c5d5839048de3f0, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/B/ef23e15caac742c896111a802b9214d4, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/B/1effae4977224fea96e128ba90c2c146] into tmpdir=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/.tmp, totalSize=47.4 K 2024-11-27T12:24:50,562 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 132920 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-27T12:24:50,562 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HStore(1541): 5dcfd84eb369d011a0951d03c984f736/A is initiating minor compaction (all files) 2024-11-27T12:24:50,562 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5dcfd84eb369d011a0951d03c984f736/A in TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736. 2024-11-27T12:24:50,562 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/A/2d76e012974b4244a2e0c7b36db4bda5, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/A/804bb279bfac454c9d73ce492a45e560, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/A/4343d343fba043e3b67f5f19dd5f4fbb, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/A/262b8f0fb1384bb19c60a861b52968d6] into tmpdir=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/.tmp, totalSize=129.8 K 2024-11-27T12:24:50,563 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736. 2024-11-27T12:24:50,563 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736. files: [hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/A/2d76e012974b4244a2e0c7b36db4bda5, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/A/804bb279bfac454c9d73ce492a45e560, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/A/4343d343fba043e3b67f5f19dd5f4fbb, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/A/262b8f0fb1384bb19c60a861b52968d6] 2024-11-27T12:24:50,563 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.Compactor(225): Compacting 722bb37ed1bb491bb719a577c007ff73, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1732710285256 2024-11-27T12:24:50,563 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2d76e012974b4244a2e0c7b36db4bda5, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1732710285256 2024-11-27T12:24:50,564 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.Compactor(225): Compacting d0eebbcdf107487e9c5d5839048de3f0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=122, earliestPutTs=1732710287459 2024-11-27T12:24:50,564 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] compactions.Compactor(225): Compacting 804bb279bfac454c9d73ce492a45e560, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=122, earliestPutTs=1732710287459 2024-11-27T12:24:50,564 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.Compactor(225): Compacting ef23e15caac742c896111a802b9214d4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1732710288645 2024-11-27T12:24:50,565 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4343d343fba043e3b67f5f19dd5f4fbb, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1732710288645 2024-11-27T12:24:50,565 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.Compactor(225): Compacting 1effae4977224fea96e128ba90c2c146, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=145, earliestPutTs=1732710289903 2024-11-27T12:24:50,565 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] compactions.Compactor(225): Compacting 262b8f0fb1384bb19c60a861b52968d6, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=145, earliestPutTs=1732710289903 2024-11-27T12:24:50,585 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=5dcfd84eb369d011a0951d03c984f736] 2024-11-27T12:24:50,588 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5dcfd84eb369d011a0951d03c984f736#B#compaction#129 average throughput is 0.60 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-27T12:24:50,589 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/.tmp/B/bb60aa00797249d1afc33648f68b319c is 50, key is test_row_0/B:col10/1732710289905/Put/seqid=0 2024-11-27T12:24:50,593 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241127e741e0cb66934df38537832f341ec6af_5dcfd84eb369d011a0951d03c984f736 store=[table=TestAcidGuarantees family=A region=5dcfd84eb369d011a0951d03c984f736] 2024-11-27T12:24:50,596 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241127e741e0cb66934df38537832f341ec6af_5dcfd84eb369d011a0951d03c984f736, store=[table=TestAcidGuarantees family=A region=5dcfd84eb369d011a0951d03c984f736] 2024-11-27T12:24:50,596 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127e741e0cb66934df38537832f341ec6af_5dcfd84eb369d011a0951d03c984f736 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=5dcfd84eb369d011a0951d03c984f736] 2024-11-27T12:24:50,604 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073741979_1155 (size=12493) 2024-11-27T12:24:50,616 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/.tmp/B/bb60aa00797249d1afc33648f68b319c as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/B/bb60aa00797249d1afc33648f68b319c 2024-11-27T12:24:50,624 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073741980_1156 (size=4469) 2024-11-27T12:24:50,627 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 5dcfd84eb369d011a0951d03c984f736/B of 5dcfd84eb369d011a0951d03c984f736 into bb60aa00797249d1afc33648f68b319c(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T12:24:50,627 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5dcfd84eb369d011a0951d03c984f736: 2024-11-27T12:24:50,627 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., storeName=5dcfd84eb369d011a0951d03c984f736/B, priority=12, startTime=1732710290559; duration=0sec 2024-11-27T12:24:50,627 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T12:24:50,627 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5dcfd84eb369d011a0951d03c984f736:B 2024-11-27T12:24:50,627 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-27T12:24:50,633 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5dcfd84eb369d011a0951d03c984f736#A#compaction#130 average throughput is 0.51 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T12:24:50,634 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48510 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-27T12:24:50,634 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/.tmp/A/07caa68475a143369e2b52fa7f5e5acf is 175, key is test_row_0/A:col10/1732710289905/Put/seqid=0 2024-11-27T12:24:50,634 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HStore(1541): 5dcfd84eb369d011a0951d03c984f736/C is initiating minor compaction (all files) 2024-11-27T12:24:50,634 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5dcfd84eb369d011a0951d03c984f736/C in TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736. 2024-11-27T12:24:50,634 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/C/c72219f77d354bc1b75f4ebde7cff5bb, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/C/53c96489235d4ef1b96340d3b844a0d3, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/C/5034c2ba00694a2fb3caa87aa991bac5, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/C/1a7819d8f9424fe38ab206b97a858095] into tmpdir=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/.tmp, totalSize=47.4 K 2024-11-27T12:24:50,636 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.Compactor(225): Compacting c72219f77d354bc1b75f4ebde7cff5bb, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1732710285256 2024-11-27T12:24:50,639 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.Compactor(225): Compacting 53c96489235d4ef1b96340d3b844a0d3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=122, earliestPutTs=1732710287459 2024-11-27T12:24:50,639 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.Compactor(225): Compacting 5034c2ba00694a2fb3caa87aa991bac5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1732710288645 2024-11-27T12:24:50,640 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.Compactor(225): Compacting 1a7819d8f9424fe38ab206b97a858095, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=145, earliestPutTs=1732710289903 2024-11-27T12:24:50,646 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073741981_1157 (size=31447) 2024-11-27T12:24:50,657 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/.tmp/A/07caa68475a143369e2b52fa7f5e5acf as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/A/07caa68475a143369e2b52fa7f5e5acf 2024-11-27T12:24:50,663 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5dcfd84eb369d011a0951d03c984f736#C#compaction#131 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T12:24:50,664 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/.tmp/C/7b0367c24548407389576730d8bc4254 is 50, key is test_row_0/C:col10/1732710289905/Put/seqid=0 2024-11-27T12:24:50,664 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 5dcfd84eb369d011a0951d03c984f736/A of 5dcfd84eb369d011a0951d03c984f736 into 07caa68475a143369e2b52fa7f5e5acf(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T12:24:50,664 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5dcfd84eb369d011a0951d03c984f736: 2024-11-27T12:24:50,664 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., storeName=5dcfd84eb369d011a0951d03c984f736/A, priority=12, startTime=1732710290559; duration=0sec 2024-11-27T12:24:50,665 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T12:24:50,665 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5dcfd84eb369d011a0951d03c984f736:A 2024-11-27T12:24:50,677 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073741982_1158 (size=12493) 2024-11-27T12:24:50,684 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/.tmp/C/7b0367c24548407389576730d8bc4254 as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/C/7b0367c24548407389576730d8bc4254 2024-11-27T12:24:50,694 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 5dcfd84eb369d011a0951d03c984f736/C of 5dcfd84eb369d011a0951d03c984f736 into 7b0367c24548407389576730d8bc4254(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T12:24:50,695 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5dcfd84eb369d011a0951d03c984f736: 2024-11-27T12:24:50,695 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., storeName=5dcfd84eb369d011a0951d03c984f736/C, priority=12, startTime=1732710290559; duration=0sec 2024-11-27T12:24:50,695 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T12:24:50,695 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5dcfd84eb369d011a0951d03c984f736:C 2024-11-27T12:24:50,720 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T12:24:50,720 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5dcfd84eb369d011a0951d03c984f736 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-27T12:24:50,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(8855): Flush requested on 5dcfd84eb369d011a0951d03c984f736 2024-11-27T12:24:50,735 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127a1b5d566004a4b3fa802ed2d08bd677a_5dcfd84eb369d011a0951d03c984f736 is 50, key is test_row_0/A:col10/1732710290718/Put/seqid=0 2024-11-27T12:24:50,746 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:50,746 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073741983_1159 (size=14794) 2024-11-27T12:24:50,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44028 deadline: 1732710350745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:50,746 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:50,746 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:50,746 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:24:50,747 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:50,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44010 deadline: 1732710350746, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:50,747 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:50,747 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:50,747 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:24:50,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-27T12:24:50,749 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T12:24:50,749 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-27T12:24:50,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] procedure2.ProcedureExecutor(1139): Stored pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees 2024-11-27T12:24:50,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-27T12:24:50,751 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T12:24:50,752 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T12:24:50,752 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=47, ppid=46, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T12:24:50,755 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:50,763 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127a1b5d566004a4b3fa802ed2d08bd677a_5dcfd84eb369d011a0951d03c984f736 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127a1b5d566004a4b3fa802ed2d08bd677a_5dcfd84eb369d011a0951d03c984f736 2024-11-27T12:24:50,764 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/.tmp/A/267bacd1d2ce460a873eb7cd2ac4eb3e, store: [table=TestAcidGuarantees family=A region=5dcfd84eb369d011a0951d03c984f736] 2024-11-27T12:24:50,767 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/.tmp/A/267bacd1d2ce460a873eb7cd2ac4eb3e is 175, key is test_row_0/A:col10/1732710290718/Put/seqid=0 2024-11-27T12:24:50,785 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073741984_1160 (size=39749) 2024-11-27T12:24:50,851 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:50,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44028 deadline: 1732710350848, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:50,851 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:50,852 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:50,852 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:24:50,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-27T12:24:50,859 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:50,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44010 deadline: 1732710350857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:50,860 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:50,860 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:50,860 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:24:50,904 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-27T12:24:50,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736. 2024-11-27T12:24:50,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736. as already flushing 2024-11-27T12:24:50,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736. 2024-11-27T12:24:50,904 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 5dcfd84eb369d011a0951d03c984f736, NAME => 'TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:24:50,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 5dcfd84eb369d011a0951d03c984f736, NAME => 'TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:24:50,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5dcfd84eb369d011a0951d03c984f736, NAME => 'TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5dcfd84eb369d011a0951d03c984f736, NAME => 'TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:24:51,057 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-27T12:24:51,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736. 2024-11-27T12:24:51,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736. as already flushing 2024-11-27T12:24:51,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736. 2024-11-27T12:24:51,057 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 5dcfd84eb369d011a0951d03c984f736, NAME => 'TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:24:51,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 5dcfd84eb369d011a0951d03c984f736, NAME => 'TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:24:51,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5dcfd84eb369d011a0951d03c984f736, NAME => 'TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5dcfd84eb369d011a0951d03c984f736, NAME => 'TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:24:51,061 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:51,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44028 deadline: 1732710351058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:51,062 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:51,062 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:51,062 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:24:51,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-27T12:24:51,073 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:51,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44010 deadline: 1732710351070, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:51,075 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:51,075 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:51,075 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:24:51,186 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=175, memsize=53.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/.tmp/A/267bacd1d2ce460a873eb7cd2ac4eb3e 2024-11-27T12:24:51,199 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/.tmp/B/2d24a83b91ee40dbab6ea579fd6eb9f2 is 50, key is test_row_0/B:col10/1732710290718/Put/seqid=0 2024-11-27T12:24:51,210 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-27T12:24:51,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736. 2024-11-27T12:24:51,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736. as already flushing 2024-11-27T12:24:51,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736. 2024-11-27T12:24:51,211 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 5dcfd84eb369d011a0951d03c984f736, NAME => 'TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:24:51,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 5dcfd84eb369d011a0951d03c984f736, NAME => 'TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:24:51,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5dcfd84eb369d011a0951d03c984f736, NAME => 'TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5dcfd84eb369d011a0951d03c984f736, NAME => 'TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:24:51,234 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073741985_1161 (size=12151) 2024-11-27T12:24:51,365 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-27T12:24:51,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736. 2024-11-27T12:24:51,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736. as already flushing 2024-11-27T12:24:51,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736. 2024-11-27T12:24:51,365 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 5dcfd84eb369d011a0951d03c984f736, NAME => 'TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:24:51,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 5dcfd84eb369d011a0951d03c984f736, NAME => 'TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:24:51,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5dcfd84eb369d011a0951d03c984f736, NAME => 'TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5dcfd84eb369d011a0951d03c984f736, NAME => 'TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:24:51,369 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:51,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44028 deadline: 1732710351368, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:51,369 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:51,369 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:51,369 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:24:51,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-27T12:24:51,390 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:51,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44010 deadline: 1732710351388, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:51,390 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:51,391 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:51,391 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:24:51,517 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-27T12:24:51,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736. 2024-11-27T12:24:51,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736. as already flushing 2024-11-27T12:24:51,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736. 2024-11-27T12:24:51,518 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 5dcfd84eb369d011a0951d03c984f736, NAME => 'TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:24:51,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 5dcfd84eb369d011a0951d03c984f736, NAME => 'TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:24:51,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5dcfd84eb369d011a0951d03c984f736, NAME => 'TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5dcfd84eb369d011a0951d03c984f736, NAME => 'TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:24:51,634 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=175 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/.tmp/B/2d24a83b91ee40dbab6ea579fd6eb9f2 2024-11-27T12:24:51,642 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/.tmp/C/901883a452494d61a2c7930a51105060 is 50, key is test_row_0/C:col10/1732710290718/Put/seqid=0 2024-11-27T12:24:51,646 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073741986_1162 (size=12151) 2024-11-27T12:24:51,647 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=175 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/.tmp/C/901883a452494d61a2c7930a51105060 2024-11-27T12:24:51,652 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/.tmp/A/267bacd1d2ce460a873eb7cd2ac4eb3e as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/A/267bacd1d2ce460a873eb7cd2ac4eb3e 2024-11-27T12:24:51,661 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/A/267bacd1d2ce460a873eb7cd2ac4eb3e, entries=200, sequenceid=175, filesize=38.8 K 2024-11-27T12:24:51,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:51,666 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/.tmp/B/2d24a83b91ee40dbab6ea579fd6eb9f2 as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/B/2d24a83b91ee40dbab6ea579fd6eb9f2 2024-11-27T12:24:51,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:51,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:51,671 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-27T12:24:51,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736. 2024-11-27T12:24:51,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736. as already flushing 2024-11-27T12:24:51,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736. 2024-11-27T12:24:51,671 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 5dcfd84eb369d011a0951d03c984f736, NAME => 'TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:24:51,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 5dcfd84eb369d011a0951d03c984f736, NAME => 'TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:24:51,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5dcfd84eb369d011a0951d03c984f736, NAME => 'TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5dcfd84eb369d011a0951d03c984f736, NAME => 'TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:24:51,682 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/B/2d24a83b91ee40dbab6ea579fd6eb9f2, entries=150, sequenceid=175, filesize=11.9 K 2024-11-27T12:24:51,683 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/.tmp/C/901883a452494d61a2c7930a51105060 as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/C/901883a452494d61a2c7930a51105060 2024-11-27T12:24:51,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:51,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:51,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:51,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:51,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:51,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:51,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:51,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:51,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:51,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:51,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:51,691 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/C/901883a452494d61a2c7930a51105060, entries=150, sequenceid=175, filesize=11.9 K 2024-11-27T12:24:51,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:51,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:51,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:51,692 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=46.96 KB/48090 for 5dcfd84eb369d011a0951d03c984f736 in 972ms, sequenceid=175, compaction requested=false 2024-11-27T12:24:51,692 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5dcfd84eb369d011a0951d03c984f736: 2024-11-27T12:24:51,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:51,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:51,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:51,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:51,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:51,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:51,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:51,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:51,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:51,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:51,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:51,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:51,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:51,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:51,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:51,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:51,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:51,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:51,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:51,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:51,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:51,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:51,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:51,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:51,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:51,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:51,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:51,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:51,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:51,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:51,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:51,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:51,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:51,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:51,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:51,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:51,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:51,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:51,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:51,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:51,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:51,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:51,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:51,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:51,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:51,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:51,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:51,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:51,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:51,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:51,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:51,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:51,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:51,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:51,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:51,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:51,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:51,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:51,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:51,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:51,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:51,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:51,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:51,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:51,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:51,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:51,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:51,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:51,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:51,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:51,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:51,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:51,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:51,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:51,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:51,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:51,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:51,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:51,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:51,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:51,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:51,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:51,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:51,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:51,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:51,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:51,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:51,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:51,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:51,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:51,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:51,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:51,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:51,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:51,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:51,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:51,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:51,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:51,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:51,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:51,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:51,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:51,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:51,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:51,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:51,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:51,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:51,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:51,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:51,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:51,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:51,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:51,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:51,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:51,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:51,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:51,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:51,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:51,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:51,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:51,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(8855): Flush requested on 5dcfd84eb369d011a0951d03c984f736 2024-11-27T12:24:51,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:51,730 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T12:24:51,730 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5dcfd84eb369d011a0951d03c984f736 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-27T12:24:51,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:51,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:51,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:51,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:51,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:51,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:51,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:51,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:51,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:51,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:51,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:51,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:51,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:51,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:51,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:51,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:51,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:51,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:51,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:51,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:51,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:51,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:51,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:51,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:51,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:51,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:51,744 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411277d5a9ee263bc4296aabd2b994c5bd397_5dcfd84eb369d011a0951d03c984f736 is 50, key is test_row_0/A:col10/1732710291728/Put/seqid=0 2024-11-27T12:24:51,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:51,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:51,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:51,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:51,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:51,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:51,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:51,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:51,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:51,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:51,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:51,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:51,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:51,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:51,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:51,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:51,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:51,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:51,757 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073741987_1163 (size=12304) 2024-11-27T12:24:51,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:51,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:51,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:51,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:51,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:51,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:51,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:51,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:51,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:51,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:51,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:51,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:51,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:51,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:51,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:51,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:51,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:51,764 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:51,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:51,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:51,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:51,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:51,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:51,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:51,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:51,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:51,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:51,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:51,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:51,775 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411277d5a9ee263bc4296aabd2b994c5bd397_5dcfd84eb369d011a0951d03c984f736 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411277d5a9ee263bc4296aabd2b994c5bd397_5dcfd84eb369d011a0951d03c984f736 2024-11-27T12:24:51,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:51,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:51,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:51,776 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/.tmp/A/317c7fd53e3c42d5a0f2dee60904c9dc, store: [table=TestAcidGuarantees family=A region=5dcfd84eb369d011a0951d03c984f736] 2024-11-27T12:24:51,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:51,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:51,777 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/.tmp/A/317c7fd53e3c42d5a0f2dee60904c9dc is 175, key is test_row_0/A:col10/1732710291728/Put/seqid=0 2024-11-27T12:24:51,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:51,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:51,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:51,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:51,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:51,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:51,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:51,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:51,781 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073741988_1164 (size=31101) 2024-11-27T12:24:51,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:51,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:51,782 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=186, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/.tmp/A/317c7fd53e3c42d5a0f2dee60904c9dc 2024-11-27T12:24:51,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:51,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:51,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:51,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:51,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:51,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:51,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:51,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:51,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:51,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:51,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:51,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:51,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:51,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:51,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:51,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:51,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:51,792 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/.tmp/B/034f8578fbab48268a0c473d3c8ada72 is 50, key is test_row_0/B:col10/1732710291728/Put/seqid=0 2024-11-27T12:24:51,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:51,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:51,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:51,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:51,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:51,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:51,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:51,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:51,825 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073741989_1165 (size=9757) 2024-11-27T12:24:51,825 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-27T12:24:51,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736. 2024-11-27T12:24:51,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736. as already flushing 2024-11-27T12:24:51,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736. 2024-11-27T12:24:51,826 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 5dcfd84eb369d011a0951d03c984f736, NAME => 'TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:24:51,826 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 5dcfd84eb369d011a0951d03c984f736, NAME => 'TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:24:51,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5dcfd84eb369d011a0951d03c984f736, NAME => 'TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5dcfd84eb369d011a0951d03c984f736, NAME => 'TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:24:51,831 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:51,831 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:51,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44012 deadline: 1732710351827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:51,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44052 deadline: 1732710351829, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:51,832 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:51,833 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:51,833 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:24:51,833 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:51,834 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:51,834 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:24:51,836 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:51,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44064 deadline: 1732710351832, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:51,837 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:51,837 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:51,837 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:24:51,882 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:51,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44028 deadline: 1732710351878, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:51,883 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:51,883 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:51,883 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:24:51,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-27T12:24:51,899 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:51,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44010 deadline: 1732710351898, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:51,900 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:51,900 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:51,900 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:24:51,939 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:51,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44012 deadline: 1732710351938, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:51,940 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:51,940 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:51,940 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:51,940 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:24:51,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44052 deadline: 1732710351938, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:51,940 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:51,941 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:51,941 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:24:51,950 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:51,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44064 deadline: 1732710351948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:51,951 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:51,952 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:51,952 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:24:51,978 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-27T12:24:51,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736. 2024-11-27T12:24:51,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736. as already flushing 2024-11-27T12:24:51,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736. 2024-11-27T12:24:51,979 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 5dcfd84eb369d011a0951d03c984f736, NAME => 'TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:24:51,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 5dcfd84eb369d011a0951d03c984f736, NAME => 'TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:24:51,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5dcfd84eb369d011a0951d03c984f736, NAME => 'TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5dcfd84eb369d011a0951d03c984f736, NAME => 'TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:24:52,132 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-27T12:24:52,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736. 2024-11-27T12:24:52,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736. as already flushing 2024-11-27T12:24:52,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736. 2024-11-27T12:24:52,132 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 5dcfd84eb369d011a0951d03c984f736, NAME => 'TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:24:52,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 5dcfd84eb369d011a0951d03c984f736, NAME => 'TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:24:52,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5dcfd84eb369d011a0951d03c984f736, NAME => 'TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5dcfd84eb369d011a0951d03c984f736, NAME => 'TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:24:52,150 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:52,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44012 deadline: 1732710352147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:52,151 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:52,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44052 deadline: 1732710352148, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:52,151 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:52,151 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:52,151 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:52,151 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:52,151 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:24:52,151 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:24:52,160 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:52,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44064 deadline: 1732710352158, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:52,161 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:52,161 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:52,161 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:24:52,226 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=186 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/.tmp/B/034f8578fbab48268a0c473d3c8ada72 2024-11-27T12:24:52,236 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/.tmp/C/331b562357eb46e8aab3dc87e8b61d65 is 50, key is test_row_0/C:col10/1732710291728/Put/seqid=0 2024-11-27T12:24:52,246 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073741990_1166 (size=9757) 2024-11-27T12:24:52,284 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-27T12:24:52,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736. 2024-11-27T12:24:52,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736. as already flushing 2024-11-27T12:24:52,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736. 2024-11-27T12:24:52,285 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 5dcfd84eb369d011a0951d03c984f736, NAME => 'TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:24:52,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 5dcfd84eb369d011a0951d03c984f736, NAME => 'TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:24:52,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5dcfd84eb369d011a0951d03c984f736, NAME => 'TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5dcfd84eb369d011a0951d03c984f736, NAME => 'TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:24:52,437 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-27T12:24:52,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736. 2024-11-27T12:24:52,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736. as already flushing 2024-11-27T12:24:52,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736. 2024-11-27T12:24:52,438 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 5dcfd84eb369d011a0951d03c984f736, NAME => 'TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:24:52,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 5dcfd84eb369d011a0951d03c984f736, NAME => 'TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:24:52,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5dcfd84eb369d011a0951d03c984f736, NAME => 'TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5dcfd84eb369d011a0951d03c984f736, NAME => 'TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:24:52,460 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:52,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44052 deadline: 1732710352458, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:52,460 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:52,460 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:52,460 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:52,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44012 deadline: 1732710352458, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:52,460 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:24:52,461 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:52,461 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:52,461 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:24:52,471 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:52,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44064 deadline: 1732710352468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:52,472 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:52,472 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:52,472 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:24:52,590 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-27T12:24:52,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736. 2024-11-27T12:24:52,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736. as already flushing 2024-11-27T12:24:52,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736. 2024-11-27T12:24:52,591 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 5dcfd84eb369d011a0951d03c984f736, NAME => 'TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:24:52,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 5dcfd84eb369d011a0951d03c984f736, NAME => 'TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:24:52,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5dcfd84eb369d011a0951d03c984f736, NAME => 'TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5dcfd84eb369d011a0951d03c984f736, NAME => 'TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:24:52,647 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=186 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/.tmp/C/331b562357eb46e8aab3dc87e8b61d65 2024-11-27T12:24:52,655 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/.tmp/A/317c7fd53e3c42d5a0f2dee60904c9dc as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/A/317c7fd53e3c42d5a0f2dee60904c9dc 2024-11-27T12:24:52,660 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/A/317c7fd53e3c42d5a0f2dee60904c9dc, entries=150, sequenceid=186, filesize=30.4 K 2024-11-27T12:24:52,661 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/.tmp/B/034f8578fbab48268a0c473d3c8ada72 as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/B/034f8578fbab48268a0c473d3c8ada72 2024-11-27T12:24:52,665 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/B/034f8578fbab48268a0c473d3c8ada72, entries=100, sequenceid=186, filesize=9.5 K 2024-11-27T12:24:52,666 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/.tmp/C/331b562357eb46e8aab3dc87e8b61d65 as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/C/331b562357eb46e8aab3dc87e8b61d65 2024-11-27T12:24:52,672 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/C/331b562357eb46e8aab3dc87e8b61d65, entries=100, sequenceid=186, filesize=9.5 K 2024-11-27T12:24:52,673 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 5dcfd84eb369d011a0951d03c984f736 in 943ms, sequenceid=186, compaction requested=true 2024-11-27T12:24:52,673 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5dcfd84eb369d011a0951d03c984f736: 2024-11-27T12:24:52,674 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5dcfd84eb369d011a0951d03c984f736:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T12:24:52,674 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T12:24:52,674 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T12:24:52,674 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5dcfd84eb369d011a0951d03c984f736:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T12:24:52,674 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T12:24:52,674 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5dcfd84eb369d011a0951d03c984f736:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T12:24:52,674 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T12:24:52,674 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T12:24:52,675 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102297 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T12:24:52,675 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HStore(1541): 5dcfd84eb369d011a0951d03c984f736/A is initiating minor compaction (all files) 2024-11-27T12:24:52,675 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5dcfd84eb369d011a0951d03c984f736/A in TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736. 2024-11-27T12:24:52,676 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/A/07caa68475a143369e2b52fa7f5e5acf, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/A/267bacd1d2ce460a873eb7cd2ac4eb3e, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/A/317c7fd53e3c42d5a0f2dee60904c9dc] into tmpdir=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/.tmp, totalSize=99.9 K 2024-11-27T12:24:52,676 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736. 2024-11-27T12:24:52,676 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736. files: [hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/A/07caa68475a143369e2b52fa7f5e5acf, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/A/267bacd1d2ce460a873eb7cd2ac4eb3e, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/A/317c7fd53e3c42d5a0f2dee60904c9dc] 2024-11-27T12:24:52,676 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34401 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T12:24:52,676 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HStore(1541): 5dcfd84eb369d011a0951d03c984f736/B is initiating minor compaction (all files) 2024-11-27T12:24:52,677 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5dcfd84eb369d011a0951d03c984f736/B in TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736. 2024-11-27T12:24:52,677 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/B/bb60aa00797249d1afc33648f68b319c, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/B/2d24a83b91ee40dbab6ea579fd6eb9f2, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/B/034f8578fbab48268a0c473d3c8ada72] into tmpdir=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/.tmp, totalSize=33.6 K 2024-11-27T12:24:52,677 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] compactions.Compactor(225): Compacting 07caa68475a143369e2b52fa7f5e5acf, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=145, earliestPutTs=1732710289903 2024-11-27T12:24:52,677 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] compactions.Compactor(225): Compacting 267bacd1d2ce460a873eb7cd2ac4eb3e, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=175, earliestPutTs=1732710290069 2024-11-27T12:24:52,677 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.Compactor(225): Compacting bb60aa00797249d1afc33648f68b319c, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=145, earliestPutTs=1732710289903 2024-11-27T12:24:52,678 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.Compactor(225): Compacting 2d24a83b91ee40dbab6ea579fd6eb9f2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=175, earliestPutTs=1732710290069 2024-11-27T12:24:52,678 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] compactions.Compactor(225): Compacting 317c7fd53e3c42d5a0f2dee60904c9dc, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=186, earliestPutTs=1732710290741 2024-11-27T12:24:52,678 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.Compactor(225): Compacting 034f8578fbab48268a0c473d3c8ada72, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=186, earliestPutTs=1732710290744 2024-11-27T12:24:52,698 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5dcfd84eb369d011a0951d03c984f736#B#compaction#138 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T12:24:52,699 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/.tmp/B/1d6635e9fbaf47b28517ad2324e0f016 is 50, key is test_row_0/B:col10/1732710291728/Put/seqid=0 2024-11-27T12:24:52,702 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=5dcfd84eb369d011a0951d03c984f736] 2024-11-27T12:24:52,709 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073741991_1167 (size=12595) 2024-11-27T12:24:52,717 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411275f3b3846c58e407bb24768eb3d351e31_5dcfd84eb369d011a0951d03c984f736 store=[table=TestAcidGuarantees family=A region=5dcfd84eb369d011a0951d03c984f736] 2024-11-27T12:24:52,720 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411275f3b3846c58e407bb24768eb3d351e31_5dcfd84eb369d011a0951d03c984f736, store=[table=TestAcidGuarantees family=A region=5dcfd84eb369d011a0951d03c984f736] 2024-11-27T12:24:52,721 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411275f3b3846c58e407bb24768eb3d351e31_5dcfd84eb369d011a0951d03c984f736 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=5dcfd84eb369d011a0951d03c984f736] 2024-11-27T12:24:52,722 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/.tmp/B/1d6635e9fbaf47b28517ad2324e0f016 as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/B/1d6635e9fbaf47b28517ad2324e0f016 2024-11-27T12:24:52,730 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5dcfd84eb369d011a0951d03c984f736/B of 5dcfd84eb369d011a0951d03c984f736 into 1d6635e9fbaf47b28517ad2324e0f016(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T12:24:52,730 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5dcfd84eb369d011a0951d03c984f736: 2024-11-27T12:24:52,731 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., storeName=5dcfd84eb369d011a0951d03c984f736/B, priority=13, startTime=1732710292674; duration=0sec 2024-11-27T12:24:52,731 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T12:24:52,731 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5dcfd84eb369d011a0951d03c984f736:B 2024-11-27T12:24:52,731 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T12:24:52,735 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34401 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T12:24:52,735 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HStore(1541): 5dcfd84eb369d011a0951d03c984f736/C is initiating minor compaction (all files) 2024-11-27T12:24:52,735 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5dcfd84eb369d011a0951d03c984f736/C in TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736. 2024-11-27T12:24:52,735 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/C/7b0367c24548407389576730d8bc4254, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/C/901883a452494d61a2c7930a51105060, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/C/331b562357eb46e8aab3dc87e8b61d65] into tmpdir=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/.tmp, totalSize=33.6 K 2024-11-27T12:24:52,736 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.Compactor(225): Compacting 7b0367c24548407389576730d8bc4254, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=145, earliestPutTs=1732710289903 2024-11-27T12:24:52,737 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.Compactor(225): Compacting 901883a452494d61a2c7930a51105060, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=175, earliestPutTs=1732710290069 2024-11-27T12:24:52,737 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.Compactor(225): Compacting 331b562357eb46e8aab3dc87e8b61d65, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=186, earliestPutTs=1732710290744 2024-11-27T12:24:52,743 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-27T12:24:52,744 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736. 2024-11-27T12:24:52,744 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2902): Flushing 5dcfd84eb369d011a0951d03c984f736 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-27T12:24:52,755 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073741992_1168 (size=4469) 2024-11-27T12:24:52,758 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5dcfd84eb369d011a0951d03c984f736#C#compaction#140 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-27T12:24:52,759 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/.tmp/C/c0deea0276bd439c9c8ee7bf44c7a2e7 is 50, key is test_row_0/C:col10/1732710291728/Put/seqid=0 2024-11-27T12:24:52,760 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5dcfd84eb369d011a0951d03c984f736#A#compaction#139 average throughput is 0.42 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T12:24:52,761 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/.tmp/A/9a7a57745fe8463e8ad9140f48e99ee4 is 175, key is test_row_0/A:col10/1732710291728/Put/seqid=0 2024-11-27T12:24:52,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411272dd5ebe1ef434bcabdcee8ee3f7298dc_5dcfd84eb369d011a0951d03c984f736 is 50, key is test_row_0/A:col10/1732710291828/Put/seqid=0 2024-11-27T12:24:52,793 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073741994_1170 (size=31656) 2024-11-27T12:24:52,801 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073741993_1169 (size=12595) 2024-11-27T12:24:52,810 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073741995_1171 (size=12304) 2024-11-27T12:24:52,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:52,812 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/.tmp/C/c0deea0276bd439c9c8ee7bf44c7a2e7 as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/C/c0deea0276bd439c9c8ee7bf44c7a2e7 2024-11-27T12:24:52,817 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411272dd5ebe1ef434bcabdcee8ee3f7298dc_5dcfd84eb369d011a0951d03c984f736 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411272dd5ebe1ef434bcabdcee8ee3f7298dc_5dcfd84eb369d011a0951d03c984f736 2024-11-27T12:24:52,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/.tmp/A/22bf183acea34b48b397980f127c08f2, store: [table=TestAcidGuarantees family=A region=5dcfd84eb369d011a0951d03c984f736] 2024-11-27T12:24:52,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/.tmp/A/22bf183acea34b48b397980f127c08f2 is 175, key is test_row_0/A:col10/1732710291828/Put/seqid=0 2024-11-27T12:24:52,822 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5dcfd84eb369d011a0951d03c984f736/C of 5dcfd84eb369d011a0951d03c984f736 into c0deea0276bd439c9c8ee7bf44c7a2e7(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T12:24:52,822 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5dcfd84eb369d011a0951d03c984f736: 2024-11-27T12:24:52,822 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., storeName=5dcfd84eb369d011a0951d03c984f736/C, priority=13, startTime=1732710292674; duration=0sec 2024-11-27T12:24:52,822 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T12:24:52,822 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5dcfd84eb369d011a0951d03c984f736:C 2024-11-27T12:24:52,830 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073741996_1172 (size=31105) 2024-11-27T12:24:52,831 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=212, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/.tmp/A/22bf183acea34b48b397980f127c08f2 2024-11-27T12:24:52,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/.tmp/B/c2b20b339ea344e8873f7c5ab9c9e309 is 50, key is test_row_0/B:col10/1732710291828/Put/seqid=0 2024-11-27T12:24:52,865 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073741997_1173 (size=12151) 2024-11-27T12:24:52,867 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=212 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/.tmp/B/c2b20b339ea344e8873f7c5ab9c9e309 2024-11-27T12:24:52,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/.tmp/C/5663d761d6cc43b8be1a74e5d690723d is 50, key is test_row_0/C:col10/1732710291828/Put/seqid=0 2024-11-27T12:24:52,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-27T12:24:52,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(8855): Flush requested on 5dcfd84eb369d011a0951d03c984f736 2024-11-27T12:24:52,901 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736. as already flushing 2024-11-27T12:24:52,913 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073741998_1174 (size=12151) 2024-11-27T12:24:52,914 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=212 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/.tmp/C/5663d761d6cc43b8be1a74e5d690723d 2024-11-27T12:24:52,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/.tmp/A/22bf183acea34b48b397980f127c08f2 as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/A/22bf183acea34b48b397980f127c08f2 2024-11-27T12:24:52,931 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/A/22bf183acea34b48b397980f127c08f2, entries=150, sequenceid=212, filesize=30.4 K 2024-11-27T12:24:52,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/.tmp/B/c2b20b339ea344e8873f7c5ab9c9e309 as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/B/c2b20b339ea344e8873f7c5ab9c9e309 2024-11-27T12:24:52,941 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:52,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44028 deadline: 1732710352938, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:52,941 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:52,941 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:52,941 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:24:52,943 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/B/c2b20b339ea344e8873f7c5ab9c9e309, entries=150, sequenceid=212, filesize=11.9 K 2024-11-27T12:24:52,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/.tmp/C/5663d761d6cc43b8be1a74e5d690723d as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/C/5663d761d6cc43b8be1a74e5d690723d 2024-11-27T12:24:52,949 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:52,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44010 deadline: 1732710352941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:52,950 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:52,950 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:52,950 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:24:52,957 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/C/5663d761d6cc43b8be1a74e5d690723d, entries=150, sequenceid=212, filesize=11.9 K 2024-11-27T12:24:52,959 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 5dcfd84eb369d011a0951d03c984f736 in 215ms, sequenceid=212, compaction requested=false 2024-11-27T12:24:52,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2603): Flush status journal for 5dcfd84eb369d011a0951d03c984f736: 2024-11-27T12:24:52,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736. 2024-11-27T12:24:52,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=47 2024-11-27T12:24:52,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.HMaster(4169): Remote procedure done, pid=47 2024-11-27T12:24:52,966 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=47, resume processing ppid=46 2024-11-27T12:24:52,966 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=47, ppid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2090 sec 2024-11-27T12:24:52,969 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees in 2.2180 sec 2024-11-27T12:24:52,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(8855): Flush requested on 5dcfd84eb369d011a0951d03c984f736 2024-11-27T12:24:52,973 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T12:24:52,973 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5dcfd84eb369d011a0951d03c984f736 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-27T12:24:53,006 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112738e2bb034fe944338de9d02abdb8e5ea_5dcfd84eb369d011a0951d03c984f736 is 50, key is test_row_0/A:col10/1732710292908/Put/seqid=0 2024-11-27T12:24:53,042 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073741999_1175 (size=14794) 2024-11-27T12:24:53,050 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:53,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44028 deadline: 1732710353048, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:53,051 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:53,051 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:53,051 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:24:53,055 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:53,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44064 deadline: 1732710353049, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:53,056 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:53,056 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:53,056 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:24:53,059 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:53,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44012 deadline: 1732710353050, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:53,060 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:53,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44052 deadline: 1732710353050, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:53,060 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:53,060 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:53,060 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:24:53,060 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:53,060 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:53,060 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:24:53,061 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:53,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44010 deadline: 1732710353058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:53,064 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:53,064 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:53,064 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:24:53,159 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:53,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44064 deadline: 1732710353158, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:53,159 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:53,160 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:53,160 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:24:53,170 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:53,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44052 deadline: 1732710353169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:53,170 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:53,170 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:53,171 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:53,171 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:24:53,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44012 deadline: 1732710353169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:53,171 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:53,171 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:53,171 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:24:53,200 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/.tmp/A/9a7a57745fe8463e8ad9140f48e99ee4 as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/A/9a7a57745fe8463e8ad9140f48e99ee4 2024-11-27T12:24:53,210 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5dcfd84eb369d011a0951d03c984f736/A of 5dcfd84eb369d011a0951d03c984f736 into 9a7a57745fe8463e8ad9140f48e99ee4(size=30.9 K), total size for store is 61.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T12:24:53,210 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5dcfd84eb369d011a0951d03c984f736: 2024-11-27T12:24:53,210 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., storeName=5dcfd84eb369d011a0951d03c984f736/A, priority=13, startTime=1732710292674; duration=0sec 2024-11-27T12:24:53,210 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T12:24:53,211 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5dcfd84eb369d011a0951d03c984f736:A 2024-11-27T12:24:53,260 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:53,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44028 deadline: 1732710353258, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:53,263 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:53,263 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:53,263 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:24:53,272 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:53,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44010 deadline: 1732710353268, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:53,273 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:53,273 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:53,273 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:24:53,368 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:53,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44064 deadline: 1732710353368, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:53,369 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:53,369 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:53,369 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:24:53,382 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:53,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44012 deadline: 1732710353379, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:53,382 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:53,382 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:53,382 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:24:53,383 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:53,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44052 deadline: 1732710353379, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:53,383 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:53,383 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:53,383 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:24:53,444 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:53,452 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112738e2bb034fe944338de9d02abdb8e5ea_5dcfd84eb369d011a0951d03c984f736 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112738e2bb034fe944338de9d02abdb8e5ea_5dcfd84eb369d011a0951d03c984f736 2024-11-27T12:24:53,453 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/.tmp/A/2d2e576b4abc4fceb474fa6c0f87aee5, store: [table=TestAcidGuarantees family=A region=5dcfd84eb369d011a0951d03c984f736] 2024-11-27T12:24:53,454 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/.tmp/A/2d2e576b4abc4fceb474fa6c0f87aee5 is 175, key is test_row_0/A:col10/1732710292908/Put/seqid=0 2024-11-27T12:24:53,489 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742000_1176 (size=39749) 2024-11-27T12:24:53,491 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=225, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/.tmp/A/2d2e576b4abc4fceb474fa6c0f87aee5 2024-11-27T12:24:53,506 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/.tmp/B/c2b805983df84a24ade534ee171115d1 is 50, key is test_row_0/B:col10/1732710292908/Put/seqid=0 2024-11-27T12:24:53,554 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742001_1177 (size=12151) 2024-11-27T12:24:53,556 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=225 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/.tmp/B/c2b805983df84a24ade534ee171115d1 2024-11-27T12:24:53,571 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:53,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44028 deadline: 1732710353569, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:53,571 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:53,571 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:53,571 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:24:53,578 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:53,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44010 deadline: 1732710353578, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:53,579 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:53,579 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:53,579 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:24:53,582 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/.tmp/C/a1a5a39a52f84dfebd19cfadb79a7fb8 is 50, key is test_row_0/C:col10/1732710292908/Put/seqid=0 2024-11-27T12:24:53,652 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742002_1178 (size=12151) 2024-11-27T12:24:53,682 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:53,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44064 deadline: 1732710353678, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:53,683 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:53,683 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:53,683 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:24:53,691 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:53,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44012 deadline: 1732710353688, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:53,692 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:53,692 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:53,692 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:24:53,693 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:53,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44052 deadline: 1732710353689, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:53,696 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:53,696 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:53,696 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:24:54,053 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=225 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/.tmp/C/a1a5a39a52f84dfebd19cfadb79a7fb8 2024-11-27T12:24:54,058 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/.tmp/A/2d2e576b4abc4fceb474fa6c0f87aee5 as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/A/2d2e576b4abc4fceb474fa6c0f87aee5 2024-11-27T12:24:54,062 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/A/2d2e576b4abc4fceb474fa6c0f87aee5, entries=200, sequenceid=225, filesize=38.8 K 2024-11-27T12:24:54,063 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/.tmp/B/c2b805983df84a24ade534ee171115d1 as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/B/c2b805983df84a24ade534ee171115d1 2024-11-27T12:24:54,067 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/B/c2b805983df84a24ade534ee171115d1, entries=150, sequenceid=225, filesize=11.9 K 2024-11-27T12:24:54,069 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/.tmp/C/a1a5a39a52f84dfebd19cfadb79a7fb8 as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/C/a1a5a39a52f84dfebd19cfadb79a7fb8 2024-11-27T12:24:54,074 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/C/a1a5a39a52f84dfebd19cfadb79a7fb8, entries=150, sequenceid=225, filesize=11.9 K 2024-11-27T12:24:54,075 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=154.31 KB/158010 for 5dcfd84eb369d011a0951d03c984f736 in 1102ms, sequenceid=225, compaction requested=true 2024-11-27T12:24:54,075 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5dcfd84eb369d011a0951d03c984f736: 2024-11-27T12:24:54,076 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T12:24:54,076 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5dcfd84eb369d011a0951d03c984f736:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T12:24:54,076 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T12:24:54,076 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5dcfd84eb369d011a0951d03c984f736:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T12:24:54,076 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T12:24:54,076 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T12:24:54,076 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5dcfd84eb369d011a0951d03c984f736:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T12:24:54,076 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T12:24:54,077 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102510 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T12:24:54,077 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HStore(1541): 5dcfd84eb369d011a0951d03c984f736/A is initiating minor compaction (all files) 2024-11-27T12:24:54,077 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5dcfd84eb369d011a0951d03c984f736/A in TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736. 2024-11-27T12:24:54,077 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/A/9a7a57745fe8463e8ad9140f48e99ee4, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/A/22bf183acea34b48b397980f127c08f2, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/A/2d2e576b4abc4fceb474fa6c0f87aee5] into tmpdir=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/.tmp, totalSize=100.1 K 2024-11-27T12:24:54,077 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736. 2024-11-27T12:24:54,077 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736. files: [hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/A/9a7a57745fe8463e8ad9140f48e99ee4, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/A/22bf183acea34b48b397980f127c08f2, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/A/2d2e576b4abc4fceb474fa6c0f87aee5] 2024-11-27T12:24:54,078 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T12:24:54,078 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HStore(1541): 5dcfd84eb369d011a0951d03c984f736/B is initiating minor compaction (all files) 2024-11-27T12:24:54,078 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5dcfd84eb369d011a0951d03c984f736/B in TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736. 2024-11-27T12:24:54,078 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/B/1d6635e9fbaf47b28517ad2324e0f016, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/B/c2b20b339ea344e8873f7c5ab9c9e309, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/B/c2b805983df84a24ade534ee171115d1] into tmpdir=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/.tmp, totalSize=36.0 K 2024-11-27T12:24:54,078 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.Compactor(225): Compacting 1d6635e9fbaf47b28517ad2324e0f016, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=186, earliestPutTs=1732710290069 2024-11-27T12:24:54,078 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9a7a57745fe8463e8ad9140f48e99ee4, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=186, earliestPutTs=1732710290069 2024-11-27T12:24:54,079 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] compactions.Compactor(225): Compacting 22bf183acea34b48b397980f127c08f2, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1732710291822 2024-11-27T12:24:54,079 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.Compactor(225): Compacting c2b20b339ea344e8873f7c5ab9c9e309, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1732710291822 2024-11-27T12:24:54,080 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2d2e576b4abc4fceb474fa6c0f87aee5, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=225, earliestPutTs=1732710292908 2024-11-27T12:24:54,080 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.Compactor(225): Compacting c2b805983df84a24ade534ee171115d1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=225, earliestPutTs=1732710292908 2024-11-27T12:24:54,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(8855): Flush requested on 5dcfd84eb369d011a0951d03c984f736 2024-11-27T12:24:54,081 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T12:24:54,081 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5dcfd84eb369d011a0951d03c984f736 3/3 column families, dataSize=161.02 KB heapSize=422.63 KB 2024-11-27T12:24:54,089 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=5dcfd84eb369d011a0951d03c984f736] 2024-11-27T12:24:54,106 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:54,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44010 deadline: 1732710354098, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:54,106 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:54,107 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:54,107 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:54,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44028 deadline: 1732710354098, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:54,107 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:24:54,107 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:54,107 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:54,107 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:24:54,115 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5dcfd84eb369d011a0951d03c984f736#B#compaction#148 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-27T12:24:54,116 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/.tmp/B/bd3c6eabab244e61a13f032655ce498e is 50, key is test_row_0/B:col10/1732710292908/Put/seqid=0 2024-11-27T12:24:54,119 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411275317aa3618b14c1cb6c6e549430d1a2b_5dcfd84eb369d011a0951d03c984f736 is 50, key is test_row_0/A:col10/1732710293048/Put/seqid=0 2024-11-27T12:24:54,129 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024112718598ff6bd31468fa7107ac9b61ea0a5_5dcfd84eb369d011a0951d03c984f736 store=[table=TestAcidGuarantees family=A region=5dcfd84eb369d011a0951d03c984f736] 2024-11-27T12:24:54,131 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024112718598ff6bd31468fa7107ac9b61ea0a5_5dcfd84eb369d011a0951d03c984f736, store=[table=TestAcidGuarantees family=A region=5dcfd84eb369d011a0951d03c984f736] 2024-11-27T12:24:54,131 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112718598ff6bd31468fa7107ac9b61ea0a5_5dcfd84eb369d011a0951d03c984f736 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=5dcfd84eb369d011a0951d03c984f736] 2024-11-27T12:24:54,146 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742003_1179 (size=12697) 2024-11-27T12:24:54,157 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742004_1180 (size=12304) 2024-11-27T12:24:54,169 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742005_1181 (size=4469) 2024-11-27T12:24:54,189 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:54,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44064 deadline: 1732710354188, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:54,190 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:54,190 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:54,190 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:24:54,202 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:54,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44012 deadline: 1732710354198, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:54,203 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:54,204 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:54,204 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:24:54,209 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:54,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44052 deadline: 1732710354208, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:54,210 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:54,210 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:54,210 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:24:54,219 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:54,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44028 deadline: 1732710354218, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:54,220 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:54,220 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:54,220 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:54,220 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:24:54,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44010 deadline: 1732710354218, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:54,221 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:54,221 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:54,221 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:24:54,431 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:54,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44028 deadline: 1732710354428, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:54,431 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:54,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44010 deadline: 1732710354428, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:54,431 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:54,431 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:54,431 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:24:54,432 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:54,432 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:54,432 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:24:54,553 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/.tmp/B/bd3c6eabab244e61a13f032655ce498e as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/B/bd3c6eabab244e61a13f032655ce498e 2024-11-27T12:24:54,558 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:54,562 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5dcfd84eb369d011a0951d03c984f736/B of 5dcfd84eb369d011a0951d03c984f736 into bd3c6eabab244e61a13f032655ce498e(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T12:24:54,562 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5dcfd84eb369d011a0951d03c984f736: 2024-11-27T12:24:54,562 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., storeName=5dcfd84eb369d011a0951d03c984f736/B, priority=13, startTime=1732710294076; duration=0sec 2024-11-27T12:24:54,562 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T12:24:54,562 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5dcfd84eb369d011a0951d03c984f736:B 2024-11-27T12:24:54,562 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T12:24:54,563 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T12:24:54,564 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HStore(1541): 5dcfd84eb369d011a0951d03c984f736/C is initiating minor compaction (all files) 2024-11-27T12:24:54,564 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5dcfd84eb369d011a0951d03c984f736/C in TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736. 2024-11-27T12:24:54,564 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/C/c0deea0276bd439c9c8ee7bf44c7a2e7, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/C/5663d761d6cc43b8be1a74e5d690723d, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/C/a1a5a39a52f84dfebd19cfadb79a7fb8] into tmpdir=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/.tmp, totalSize=36.0 K 2024-11-27T12:24:54,565 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.Compactor(225): Compacting c0deea0276bd439c9c8ee7bf44c7a2e7, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=186, earliestPutTs=1732710290069 2024-11-27T12:24:54,566 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411275317aa3618b14c1cb6c6e549430d1a2b_5dcfd84eb369d011a0951d03c984f736 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411275317aa3618b14c1cb6c6e549430d1a2b_5dcfd84eb369d011a0951d03c984f736 2024-11-27T12:24:54,568 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.Compactor(225): Compacting 5663d761d6cc43b8be1a74e5d690723d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1732710291822 2024-11-27T12:24:54,570 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/.tmp/A/da40a52557cf489391501eaeb20cba6b, store: [table=TestAcidGuarantees family=A region=5dcfd84eb369d011a0951d03c984f736] 2024-11-27T12:24:54,570 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.Compactor(225): Compacting a1a5a39a52f84dfebd19cfadb79a7fb8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=225, earliestPutTs=1732710292908 2024-11-27T12:24:54,570 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/.tmp/A/da40a52557cf489391501eaeb20cba6b is 175, key is test_row_0/A:col10/1732710293048/Put/seqid=0 2024-11-27T12:24:54,573 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5dcfd84eb369d011a0951d03c984f736#A#compaction#147 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T12:24:54,574 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/.tmp/A/976d62fcda1f4021a434eabaf16df647 is 175, key is test_row_0/A:col10/1732710292908/Put/seqid=0 2024-11-27T12:24:54,581 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742006_1182 (size=31105) 2024-11-27T12:24:54,582 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=253, memsize=53.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/.tmp/A/da40a52557cf489391501eaeb20cba6b 2024-11-27T12:24:54,602 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742007_1183 (size=31651) 2024-11-27T12:24:54,608 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5dcfd84eb369d011a0951d03c984f736#C#compaction#150 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T12:24:54,609 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/.tmp/C/0bbb910c50294a0d9629ddcb403a33e7 is 50, key is test_row_0/C:col10/1732710292908/Put/seqid=0 2024-11-27T12:24:54,612 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/.tmp/A/976d62fcda1f4021a434eabaf16df647 as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/A/976d62fcda1f4021a434eabaf16df647 2024-11-27T12:24:54,618 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5dcfd84eb369d011a0951d03c984f736/A of 5dcfd84eb369d011a0951d03c984f736 into 976d62fcda1f4021a434eabaf16df647(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T12:24:54,618 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5dcfd84eb369d011a0951d03c984f736: 2024-11-27T12:24:54,619 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., storeName=5dcfd84eb369d011a0951d03c984f736/A, priority=13, startTime=1732710294075; duration=0sec 2024-11-27T12:24:54,619 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T12:24:54,619 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5dcfd84eb369d011a0951d03c984f736:A 2024-11-27T12:24:54,623 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/.tmp/B/afdf4e6b2a0e46f1b51d52418f9df0c6 is 50, key is test_row_0/B:col10/1732710293048/Put/seqid=0 2024-11-27T12:24:54,641 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742008_1184 (size=12697) 2024-11-27T12:24:54,650 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742009_1185 (size=12151) 2024-11-27T12:24:54,653 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=253 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/.tmp/B/afdf4e6b2a0e46f1b51d52418f9df0c6 2024-11-27T12:24:54,666 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/.tmp/C/99394b71d8bd45d3a7914292b67e306c is 50, key is test_row_0/C:col10/1732710293048/Put/seqid=0 2024-11-27T12:24:54,692 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742010_1186 (size=12151) 2024-11-27T12:24:54,741 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:54,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44028 deadline: 1732710354738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:54,742 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:54,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44010 deadline: 1732710354738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:54,743 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:54,743 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:54,743 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:24:54,743 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:54,743 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:54,744 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:24:54,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-27T12:24:54,928 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T12:24:54,930 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-27T12:24:54,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] procedure2.ProcedureExecutor(1139): Stored pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees 2024-11-27T12:24:54,932 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T12:24:54,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-27T12:24:54,933 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T12:24:54,933 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=49, ppid=48, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T12:24:55,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-27T12:24:55,048 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/.tmp/C/0bbb910c50294a0d9629ddcb403a33e7 as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/C/0bbb910c50294a0d9629ddcb403a33e7 2024-11-27T12:24:55,056 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5dcfd84eb369d011a0951d03c984f736/C of 5dcfd84eb369d011a0951d03c984f736 into 0bbb910c50294a0d9629ddcb403a33e7(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T12:24:55,056 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5dcfd84eb369d011a0951d03c984f736: 2024-11-27T12:24:55,056 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., storeName=5dcfd84eb369d011a0951d03c984f736/C, priority=13, startTime=1732710294076; duration=0sec 2024-11-27T12:24:55,056 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T12:24:55,056 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5dcfd84eb369d011a0951d03c984f736:C 2024-11-27T12:24:55,085 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-27T12:24:55,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736. 2024-11-27T12:24:55,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736. as already flushing 2024-11-27T12:24:55,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736. 2024-11-27T12:24:55,086 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 5dcfd84eb369d011a0951d03c984f736, NAME => 'TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:24:55,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 5dcfd84eb369d011a0951d03c984f736, NAME => 'TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:24:55,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5dcfd84eb369d011a0951d03c984f736, NAME => 'TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5dcfd84eb369d011a0951d03c984f736, NAME => 'TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:24:55,100 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=253 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/.tmp/C/99394b71d8bd45d3a7914292b67e306c 2024-11-27T12:24:55,105 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/.tmp/A/da40a52557cf489391501eaeb20cba6b as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/A/da40a52557cf489391501eaeb20cba6b 2024-11-27T12:24:55,110 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/A/da40a52557cf489391501eaeb20cba6b, entries=150, sequenceid=253, filesize=30.4 K 2024-11-27T12:24:55,111 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/.tmp/B/afdf4e6b2a0e46f1b51d52418f9df0c6 as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/B/afdf4e6b2a0e46f1b51d52418f9df0c6 2024-11-27T12:24:55,116 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/B/afdf4e6b2a0e46f1b51d52418f9df0c6, entries=150, sequenceid=253, filesize=11.9 K 2024-11-27T12:24:55,117 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/.tmp/C/99394b71d8bd45d3a7914292b67e306c as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/C/99394b71d8bd45d3a7914292b67e306c 2024-11-27T12:24:55,121 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/C/99394b71d8bd45d3a7914292b67e306c, entries=150, sequenceid=253, filesize=11.9 K 2024-11-27T12:24:55,122 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=46.96 KB/48090 for 5dcfd84eb369d011a0951d03c984f736 in 1041ms, sequenceid=253, compaction requested=false 2024-11-27T12:24:55,122 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5dcfd84eb369d011a0951d03c984f736: 2024-11-27T12:24:55,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(8855): Flush requested on 5dcfd84eb369d011a0951d03c984f736 2024-11-27T12:24:55,199 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T12:24:55,200 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5dcfd84eb369d011a0951d03c984f736 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-27T12:24:55,222 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411277c13fda6fa4f44fe9504df085ddc48a2_5dcfd84eb369d011a0951d03c984f736 is 50, key is test_row_0/A:col10/1732710295198/Put/seqid=0 2024-11-27T12:24:55,239 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-27T12:24:55,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736. 2024-11-27T12:24:55,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736. as already flushing 2024-11-27T12:24:55,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736. 2024-11-27T12:24:55,239 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 5dcfd84eb369d011a0951d03c984f736, NAME => 'TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:24:55,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 5dcfd84eb369d011a0951d03c984f736, NAME => 'TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:24:55,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5dcfd84eb369d011a0951d03c984f736, NAME => 'TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5dcfd84eb369d011a0951d03c984f736, NAME => 'TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:24:55,243 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742011_1187 (size=14994) 2024-11-27T12:24:55,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-27T12:24:55,256 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:55,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44010 deadline: 1732710355249, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:55,256 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:55,256 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:55,256 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:24:55,258 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:55,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44064 deadline: 1732710355250, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:55,258 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:55,258 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:55,258 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:24:55,260 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:55,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44052 deadline: 1732710355255, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:55,260 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:55,260 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:55,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44012 deadline: 1732710355256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:55,260 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:55,260 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:24:55,261 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:55,261 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:55,261 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:55,261 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:24:55,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44028 deadline: 1732710355258, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:55,261 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:55,261 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:55,261 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:24:55,370 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:55,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44064 deadline: 1732710355368, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:55,370 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:55,370 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:55,370 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:55,370 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:24:55,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44052 deadline: 1732710355368, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:55,371 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:55,371 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:55,371 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:24:55,371 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:55,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44012 deadline: 1732710355368, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:55,371 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:55,371 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:55,371 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:24:55,392 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-27T12:24:55,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736. 2024-11-27T12:24:55,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736. as already flushing 2024-11-27T12:24:55,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736. 2024-11-27T12:24:55,392 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 5dcfd84eb369d011a0951d03c984f736, NAME => 'TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:24:55,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 5dcfd84eb369d011a0951d03c984f736, NAME => 'TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:24:55,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5dcfd84eb369d011a0951d03c984f736, NAME => 'TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5dcfd84eb369d011a0951d03c984f736, NAME => 'TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:24:55,545 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-27T12:24:55,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736. 2024-11-27T12:24:55,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736. as already flushing 2024-11-27T12:24:55,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736. 2024-11-27T12:24:55,545 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 5dcfd84eb369d011a0951d03c984f736, NAME => 'TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:24:55,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 5dcfd84eb369d011a0951d03c984f736, NAME => 'TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:24:55,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5dcfd84eb369d011a0951d03c984f736, NAME => 'TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5dcfd84eb369d011a0951d03c984f736, NAME => 'TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:24:55,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-27T12:24:55,581 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:55,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44064 deadline: 1732710355578, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:55,582 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:55,582 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:55,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44052 deadline: 1732710355578, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:55,582 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:55,582 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:24:55,582 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:55,582 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:55,582 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:55,582 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:24:55,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44012 deadline: 1732710355579, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:55,582 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:55,582 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:55,583 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:24:55,645 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:55,652 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411277c13fda6fa4f44fe9504df085ddc48a2_5dcfd84eb369d011a0951d03c984f736 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411277c13fda6fa4f44fe9504df085ddc48a2_5dcfd84eb369d011a0951d03c984f736 2024-11-27T12:24:55,655 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/.tmp/A/712ff11a1ce344049497c0064807926f, store: [table=TestAcidGuarantees family=A region=5dcfd84eb369d011a0951d03c984f736] 2024-11-27T12:24:55,656 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/.tmp/A/712ff11a1ce344049497c0064807926f is 175, key is test_row_0/A:col10/1732710295198/Put/seqid=0 2024-11-27T12:24:55,659 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742012_1188 (size=39949) 2024-11-27T12:24:55,703 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-27T12:24:55,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736. 2024-11-27T12:24:55,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736. as already flushing 2024-11-27T12:24:55,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736. 2024-11-27T12:24:55,703 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 5dcfd84eb369d011a0951d03c984f736, NAME => 'TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:24:55,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 5dcfd84eb369d011a0951d03c984f736, NAME => 'TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:24:55,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5dcfd84eb369d011a0951d03c984f736, NAME => 'TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5dcfd84eb369d011a0951d03c984f736, NAME => 'TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:24:55,856 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-27T12:24:55,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736. 2024-11-27T12:24:55,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736. as already flushing 2024-11-27T12:24:55,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736. 2024-11-27T12:24:55,857 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 5dcfd84eb369d011a0951d03c984f736, NAME => 'TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:24:55,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 5dcfd84eb369d011a0951d03c984f736, NAME => 'TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:24:55,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5dcfd84eb369d011a0951d03c984f736, NAME => 'TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5dcfd84eb369d011a0951d03c984f736, NAME => 'TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:24:55,890 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:55,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44052 deadline: 1732710355888, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:55,891 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:55,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44064 deadline: 1732710355888, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:55,891 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:55,891 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:55,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44012 deadline: 1732710355888, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:55,891 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:55,891 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:24:55,891 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:55,892 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:55,892 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:24:55,892 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:55,892 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:55,892 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:24:56,010 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-27T12:24:56,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736. 2024-11-27T12:24:56,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736. as already flushing 2024-11-27T12:24:56,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736. 2024-11-27T12:24:56,010 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 5dcfd84eb369d011a0951d03c984f736, NAME => 'TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:24:56,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 5dcfd84eb369d011a0951d03c984f736, NAME => 'TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:24:56,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5dcfd84eb369d011a0951d03c984f736, NAME => 'TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5dcfd84eb369d011a0951d03c984f736, NAME => 'TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:24:56,060 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=267, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/.tmp/A/712ff11a1ce344049497c0064807926f 2024-11-27T12:24:56,067 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/.tmp/B/1a7df16e395b48378b208512675e8475 is 50, key is test_row_0/B:col10/1732710295198/Put/seqid=0 2024-11-27T12:24:56,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-27T12:24:56,074 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742013_1189 (size=12301) 2024-11-27T12:24:56,164 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-27T12:24:56,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736. 2024-11-27T12:24:56,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736. as already flushing 2024-11-27T12:24:56,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736. 2024-11-27T12:24:56,164 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 5dcfd84eb369d011a0951d03c984f736, NAME => 'TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:24:56,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 5dcfd84eb369d011a0951d03c984f736, NAME => 'TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:24:56,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5dcfd84eb369d011a0951d03c984f736, NAME => 'TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5dcfd84eb369d011a0951d03c984f736, NAME => 'TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:24:56,270 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:56,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44010 deadline: 1732710356268, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:56,270 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:56,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44028 deadline: 1732710356268, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:56,270 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to 1ba5af43fd99,36251,1732710241631 for 'test_row_2' in 5dcfd84eb369d011a0951d03c984f736 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2172 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T12:24:56,271 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:56,271 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:56,271 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:24:56,271 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 1ba5af43fd99,36251,1732710241631 for 'test_row_1' in 5dcfd84eb369d011a0951d03c984f736 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2173 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T12:24:56,272 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:56,272 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:56,272 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:24:56,317 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-27T12:24:56,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736. 2024-11-27T12:24:56,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736. as already flushing 2024-11-27T12:24:56,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736. 2024-11-27T12:24:56,317 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 5dcfd84eb369d011a0951d03c984f736, NAME => 'TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:24:56,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 5dcfd84eb369d011a0951d03c984f736, NAME => 'TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:24:56,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5dcfd84eb369d011a0951d03c984f736, NAME => 'TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5dcfd84eb369d011a0951d03c984f736, NAME => 'TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:24:56,401 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:56,401 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:56,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44012 deadline: 1732710356398, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:56,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44064 deadline: 1732710356398, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:56,401 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:56,401 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:56,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44052 deadline: 1732710356398, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:56,401 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:56,401 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:56,401 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:56,401 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:24:56,401 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:24:56,401 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:56,401 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:56,402 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:24:56,470 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-27T12:24:56,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736. 2024-11-27T12:24:56,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736. as already flushing 2024-11-27T12:24:56,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736. 2024-11-27T12:24:56,471 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 5dcfd84eb369d011a0951d03c984f736, NAME => 'TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:24:56,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 5dcfd84eb369d011a0951d03c984f736, NAME => 'TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:24:56,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5dcfd84eb369d011a0951d03c984f736, NAME => 'TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5dcfd84eb369d011a0951d03c984f736, NAME => 'TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:24:56,475 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=267 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/.tmp/B/1a7df16e395b48378b208512675e8475 2024-11-27T12:24:56,483 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/.tmp/C/fb1b2473ef854d8fb92516fb1650f5b6 is 50, key is test_row_0/C:col10/1732710295198/Put/seqid=0 2024-11-27T12:24:56,499 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742014_1190 (size=12301) 2024-11-27T12:24:56,505 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=267 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/.tmp/C/fb1b2473ef854d8fb92516fb1650f5b6 2024-11-27T12:24:56,517 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/.tmp/A/712ff11a1ce344049497c0064807926f as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/A/712ff11a1ce344049497c0064807926f 2024-11-27T12:24:56,526 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/A/712ff11a1ce344049497c0064807926f, entries=200, sequenceid=267, filesize=39.0 K 2024-11-27T12:24:56,527 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/.tmp/B/1a7df16e395b48378b208512675e8475 as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/B/1a7df16e395b48378b208512675e8475 2024-11-27T12:24:56,532 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/B/1a7df16e395b48378b208512675e8475, entries=150, sequenceid=267, filesize=12.0 K 2024-11-27T12:24:56,535 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/.tmp/C/fb1b2473ef854d8fb92516fb1650f5b6 as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/C/fb1b2473ef854d8fb92516fb1650f5b6 2024-11-27T12:24:56,540 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/C/fb1b2473ef854d8fb92516fb1650f5b6, entries=150, sequenceid=267, filesize=12.0 K 2024-11-27T12:24:56,541 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 5dcfd84eb369d011a0951d03c984f736 in 1342ms, sequenceid=267, compaction requested=true 2024-11-27T12:24:56,542 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5dcfd84eb369d011a0951d03c984f736: 2024-11-27T12:24:56,542 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5dcfd84eb369d011a0951d03c984f736:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T12:24:56,542 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T12:24:56,542 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T12:24:56,542 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T12:24:56,542 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5dcfd84eb369d011a0951d03c984f736:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T12:24:56,542 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T12:24:56,543 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5dcfd84eb369d011a0951d03c984f736:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T12:24:56,543 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T12:24:56,543 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102705 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T12:24:56,543 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37149 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T12:24:56,543 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HStore(1541): 5dcfd84eb369d011a0951d03c984f736/B is initiating minor compaction (all files) 2024-11-27T12:24:56,543 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HStore(1541): 5dcfd84eb369d011a0951d03c984f736/A is initiating minor compaction (all files) 2024-11-27T12:24:56,543 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5dcfd84eb369d011a0951d03c984f736/A in TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736. 2024-11-27T12:24:56,543 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5dcfd84eb369d011a0951d03c984f736/B in TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736. 2024-11-27T12:24:56,543 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/B/bd3c6eabab244e61a13f032655ce498e, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/B/afdf4e6b2a0e46f1b51d52418f9df0c6, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/B/1a7df16e395b48378b208512675e8475] into tmpdir=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/.tmp, totalSize=36.3 K 2024-11-27T12:24:56,543 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/A/976d62fcda1f4021a434eabaf16df647, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/A/da40a52557cf489391501eaeb20cba6b, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/A/712ff11a1ce344049497c0064807926f] into tmpdir=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/.tmp, totalSize=100.3 K 2024-11-27T12:24:56,543 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736. 2024-11-27T12:24:56,543 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736. files: [hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/A/976d62fcda1f4021a434eabaf16df647, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/A/da40a52557cf489391501eaeb20cba6b, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/A/712ff11a1ce344049497c0064807926f] 2024-11-27T12:24:56,544 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] compactions.Compactor(225): Compacting 976d62fcda1f4021a434eabaf16df647, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=225, earliestPutTs=1732710292908 2024-11-27T12:24:56,544 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.Compactor(225): Compacting bd3c6eabab244e61a13f032655ce498e, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=225, earliestPutTs=1732710292908 2024-11-27T12:24:56,545 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.Compactor(225): Compacting afdf4e6b2a0e46f1b51d52418f9df0c6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1732710292998 2024-11-27T12:24:56,545 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] compactions.Compactor(225): Compacting da40a52557cf489391501eaeb20cba6b, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1732710292998 2024-11-27T12:24:56,545 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.Compactor(225): Compacting 1a7df16e395b48378b208512675e8475, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=267, earliestPutTs=1732710294096 2024-11-27T12:24:56,545 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] compactions.Compactor(225): Compacting 712ff11a1ce344049497c0064807926f, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=267, earliestPutTs=1732710294091 2024-11-27T12:24:56,553 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=5dcfd84eb369d011a0951d03c984f736] 2024-11-27T12:24:56,556 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5dcfd84eb369d011a0951d03c984f736#B#compaction#157 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-27T12:24:56,556 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241127500a4227e8a743bcab3261674e43f9ba_5dcfd84eb369d011a0951d03c984f736 store=[table=TestAcidGuarantees family=A region=5dcfd84eb369d011a0951d03c984f736] 2024-11-27T12:24:56,556 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/.tmp/B/f3e7a2edbd234f9fa8eb1b0979229419 is 50, key is test_row_0/B:col10/1732710295198/Put/seqid=0 2024-11-27T12:24:56,558 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241127500a4227e8a743bcab3261674e43f9ba_5dcfd84eb369d011a0951d03c984f736, store=[table=TestAcidGuarantees family=A region=5dcfd84eb369d011a0951d03c984f736] 2024-11-27T12:24:56,559 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127500a4227e8a743bcab3261674e43f9ba_5dcfd84eb369d011a0951d03c984f736 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=5dcfd84eb369d011a0951d03c984f736] 2024-11-27T12:24:56,578 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742015_1191 (size=12949) 2024-11-27T12:24:56,581 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742016_1192 (size=4469) 2024-11-27T12:24:56,583 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5dcfd84eb369d011a0951d03c984f736#A#compaction#156 average throughput is 0.81 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T12:24:56,583 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/.tmp/A/cef0ad5fa2554478b26473f6dd351323 is 175, key is test_row_0/A:col10/1732710295198/Put/seqid=0 2024-11-27T12:24:56,591 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742017_1193 (size=31903) 2024-11-27T12:24:56,601 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/.tmp/A/cef0ad5fa2554478b26473f6dd351323 as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/A/cef0ad5fa2554478b26473f6dd351323 2024-11-27T12:24:56,611 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5dcfd84eb369d011a0951d03c984f736/A of 5dcfd84eb369d011a0951d03c984f736 into cef0ad5fa2554478b26473f6dd351323(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T12:24:56,611 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5dcfd84eb369d011a0951d03c984f736: 2024-11-27T12:24:56,611 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., storeName=5dcfd84eb369d011a0951d03c984f736/A, priority=13, startTime=1732710296542; duration=0sec 2024-11-27T12:24:56,611 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T12:24:56,612 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5dcfd84eb369d011a0951d03c984f736:A 2024-11-27T12:24:56,612 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T12:24:56,614 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37149 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T12:24:56,614 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HStore(1541): 5dcfd84eb369d011a0951d03c984f736/C is initiating minor compaction (all files) 2024-11-27T12:24:56,614 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5dcfd84eb369d011a0951d03c984f736/C in TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736. 2024-11-27T12:24:56,614 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/C/0bbb910c50294a0d9629ddcb403a33e7, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/C/99394b71d8bd45d3a7914292b67e306c, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/C/fb1b2473ef854d8fb92516fb1650f5b6] into tmpdir=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/.tmp, totalSize=36.3 K 2024-11-27T12:24:56,615 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0bbb910c50294a0d9629ddcb403a33e7, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=225, earliestPutTs=1732710292908 2024-11-27T12:24:56,615 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] compactions.Compactor(225): Compacting 99394b71d8bd45d3a7914292b67e306c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1732710292998 2024-11-27T12:24:56,616 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] compactions.Compactor(225): Compacting fb1b2473ef854d8fb92516fb1650f5b6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=267, earliestPutTs=1732710294096 2024-11-27T12:24:56,624 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-27T12:24:56,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736. 2024-11-27T12:24:56,625 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2902): Flushing 5dcfd84eb369d011a0951d03c984f736 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-27T12:24:56,631 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5dcfd84eb369d011a0951d03c984f736#C#compaction#158 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T12:24:56,632 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/.tmp/C/2aa8553d58414fa8b3d7135027dc6059 is 50, key is test_row_0/C:col10/1732710295198/Put/seqid=0 2024-11-27T12:24:56,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127451a9c6fad334d63bcf8750b0c271ee2_5dcfd84eb369d011a0951d03c984f736 is 50, key is test_row_0/A:col10/1732710295242/Put/seqid=0 2024-11-27T12:24:56,662 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742018_1194 (size=12949) 2024-11-27T12:24:56,671 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742019_1195 (size=12454) 2024-11-27T12:24:56,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:56,676 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127451a9c6fad334d63bcf8750b0c271ee2_5dcfd84eb369d011a0951d03c984f736 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127451a9c6fad334d63bcf8750b0c271ee2_5dcfd84eb369d011a0951d03c984f736 2024-11-27T12:24:56,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/.tmp/A/a1b15381698b499a8023674f8cf9dd92, store: [table=TestAcidGuarantees family=A region=5dcfd84eb369d011a0951d03c984f736] 2024-11-27T12:24:56,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/.tmp/A/a1b15381698b499a8023674f8cf9dd92 is 175, key is test_row_0/A:col10/1732710295242/Put/seqid=0 2024-11-27T12:24:56,690 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742020_1196 (size=31255) 2024-11-27T12:24:56,982 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/.tmp/B/f3e7a2edbd234f9fa8eb1b0979229419 as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/B/f3e7a2edbd234f9fa8eb1b0979229419 2024-11-27T12:24:56,986 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5dcfd84eb369d011a0951d03c984f736/B of 5dcfd84eb369d011a0951d03c984f736 into f3e7a2edbd234f9fa8eb1b0979229419(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T12:24:56,986 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5dcfd84eb369d011a0951d03c984f736: 2024-11-27T12:24:56,986 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., storeName=5dcfd84eb369d011a0951d03c984f736/B, priority=13, startTime=1732710296542; duration=0sec 2024-11-27T12:24:56,986 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T12:24:56,986 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5dcfd84eb369d011a0951d03c984f736:B 2024-11-27T12:24:57,068 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/.tmp/C/2aa8553d58414fa8b3d7135027dc6059 as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/C/2aa8553d58414fa8b3d7135027dc6059 2024-11-27T12:24:57,074 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5dcfd84eb369d011a0951d03c984f736/C of 5dcfd84eb369d011a0951d03c984f736 into 2aa8553d58414fa8b3d7135027dc6059(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T12:24:57,074 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5dcfd84eb369d011a0951d03c984f736: 2024-11-27T12:24:57,074 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., storeName=5dcfd84eb369d011a0951d03c984f736/C, priority=13, startTime=1732710296542; duration=0sec 2024-11-27T12:24:57,074 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T12:24:57,074 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5dcfd84eb369d011a0951d03c984f736:C 2024-11-27T12:24:57,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-27T12:24:57,090 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=293, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/.tmp/A/a1b15381698b499a8023674f8cf9dd92 2024-11-27T12:24:57,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/.tmp/B/9786d1db24d145c8a2b9dbf40b7435f4 is 50, key is test_row_0/B:col10/1732710295242/Put/seqid=0 2024-11-27T12:24:57,111 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742021_1197 (size=12301) 2024-11-27T12:24:57,410 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736. as already flushing 2024-11-27T12:24:57,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(8855): Flush requested on 5dcfd84eb369d011a0951d03c984f736 2024-11-27T12:24:57,428 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:57,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44052 deadline: 1732710357425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:57,429 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:57,429 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:57,429 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:24:57,433 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:57,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44064 deadline: 1732710357428, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:57,434 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:57,434 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:57,434 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:24:57,437 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:57,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44012 deadline: 1732710357430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:57,437 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:57,437 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:57,437 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:24:57,512 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=293 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/.tmp/B/9786d1db24d145c8a2b9dbf40b7435f4 2024-11-27T12:24:57,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/.tmp/C/2f4e973b26f94a1190a7d9a5321b9bcc is 50, key is test_row_0/C:col10/1732710295242/Put/seqid=0 2024-11-27T12:24:57,538 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:57,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44052 deadline: 1732710357538, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:57,539 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:57,539 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:57,539 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:24:57,542 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:57,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44064 deadline: 1732710357538, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:57,543 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:57,543 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:57,543 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:24:57,551 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:57,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44012 deadline: 1732710357548, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:57,552 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:57,552 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:57,552 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:24:57,577 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742022_1198 (size=12301) 2024-11-27T12:24:57,750 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:57,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44052 deadline: 1732710357748, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:57,750 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:57,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44064 deadline: 1732710357748, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:57,750 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:57,750 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:57,750 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:57,750 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:24:57,750 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:57,750 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:24:57,760 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:57,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44012 deadline: 1732710357758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:57,760 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:57,760 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:57,760 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:24:57,978 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=293 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/.tmp/C/2f4e973b26f94a1190a7d9a5321b9bcc 2024-11-27T12:24:57,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/.tmp/A/a1b15381698b499a8023674f8cf9dd92 as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/A/a1b15381698b499a8023674f8cf9dd92 2024-11-27T12:24:57,987 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/A/a1b15381698b499a8023674f8cf9dd92, entries=150, sequenceid=293, filesize=30.5 K 2024-11-27T12:24:57,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/.tmp/B/9786d1db24d145c8a2b9dbf40b7435f4 as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/B/9786d1db24d145c8a2b9dbf40b7435f4 2024-11-27T12:24:57,993 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/B/9786d1db24d145c8a2b9dbf40b7435f4, entries=150, sequenceid=293, filesize=12.0 K 2024-11-27T12:24:57,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/.tmp/C/2f4e973b26f94a1190a7d9a5321b9bcc as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/C/2f4e973b26f94a1190a7d9a5321b9bcc 2024-11-27T12:24:57,998 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/C/2f4e973b26f94a1190a7d9a5321b9bcc, entries=150, sequenceid=293, filesize=12.0 K 2024-11-27T12:24:57,999 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=60.38 KB/61830 for 5dcfd84eb369d011a0951d03c984f736 in 1374ms, sequenceid=293, compaction requested=false 2024-11-27T12:24:57,999 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2603): Flush status journal for 5dcfd84eb369d011a0951d03c984f736: 2024-11-27T12:24:57,999 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736. 2024-11-27T12:24:57,999 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=49 2024-11-27T12:24:57,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.HMaster(4169): Remote procedure done, pid=49 2024-11-27T12:24:58,002 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=49, resume processing ppid=48 2024-11-27T12:24:58,002 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=49, ppid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.0670 sec 2024-11-27T12:24:58,004 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees in 3.0720 sec 2024-11-27T12:24:58,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(8855): Flush requested on 5dcfd84eb369d011a0951d03c984f736 2024-11-27T12:24:58,060 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T12:24:58,060 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5dcfd84eb369d011a0951d03c984f736 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-27T12:24:58,075 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127892323c757d84c68b1b888bb13a592cb_5dcfd84eb369d011a0951d03c984f736 is 50, key is test_row_0/A:col10/1732710298059/Put/seqid=0 2024-11-27T12:24:58,079 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742023_1199 (size=17534) 2024-11-27T12:24:58,081 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:58,085 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127892323c757d84c68b1b888bb13a592cb_5dcfd84eb369d011a0951d03c984f736 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127892323c757d84c68b1b888bb13a592cb_5dcfd84eb369d011a0951d03c984f736 2024-11-27T12:24:58,086 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/.tmp/A/b2e8fe6209fb4574b62784fa5a292246, store: [table=TestAcidGuarantees family=A region=5dcfd84eb369d011a0951d03c984f736] 2024-11-27T12:24:58,087 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/.tmp/A/b2e8fe6209fb4574b62784fa5a292246 is 175, key is test_row_0/A:col10/1732710298059/Put/seqid=0 2024-11-27T12:24:58,092 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742024_1200 (size=48639) 2024-11-27T12:24:58,133 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:58,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44052 deadline: 1732710358124, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:58,133 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:58,133 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:58,133 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:58,133 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:24:58,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44012 deadline: 1732710358124, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:58,134 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:58,134 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:58,134 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:24:58,134 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:58,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44064 deadline: 1732710358124, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:58,134 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:58,134 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:58,134 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:24:58,243 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:58,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44052 deadline: 1732710358238, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:58,243 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:58,243 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:58,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44012 deadline: 1732710358238, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:58,243 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:58,243 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:24:58,244 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:58,244 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:58,244 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:58,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44064 deadline: 1732710358238, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:58,244 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:24:58,244 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:58,244 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:58,244 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:24:58,280 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:58,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44010 deadline: 1732710358278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:58,281 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to 1ba5af43fd99,36251,1732710241631 for 'test_row_2' in 5dcfd84eb369d011a0951d03c984f736 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4182 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T12:24:58,281 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:58,281 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:58,281 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:24:58,292 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:58,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44028 deadline: 1732710358288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:58,293 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 1ba5af43fd99,36251,1732710241631 for 'test_row_1' in 5dcfd84eb369d011a0951d03c984f736 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4194 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T12:24:58,293 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:58,294 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:58,294 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:24:58,451 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:58,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44052 deadline: 1732710358448, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:58,451 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:58,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44012 deadline: 1732710358448, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:58,452 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:58,452 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:58,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44064 deadline: 1732710358448, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:58,452 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:58,452 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:24:58,452 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:58,452 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:58,452 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:24:58,452 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:58,452 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:58,452 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:24:58,493 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=309, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/.tmp/A/b2e8fe6209fb4574b62784fa5a292246 2024-11-27T12:24:58,501 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/.tmp/B/1c0751f5da2148d7b03a0f6e6870dc43 is 50, key is test_row_0/B:col10/1732710298059/Put/seqid=0 2024-11-27T12:24:58,504 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742025_1201 (size=12301) 2024-11-27T12:24:58,505 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=309 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/.tmp/B/1c0751f5da2148d7b03a0f6e6870dc43 2024-11-27T12:24:58,514 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/.tmp/C/b83f65028cfc48628a10cec0f810bb1d is 50, key is test_row_0/C:col10/1732710298059/Put/seqid=0 2024-11-27T12:24:58,518 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742026_1202 (size=12301) 2024-11-27T12:24:58,519 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=309 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/.tmp/C/b83f65028cfc48628a10cec0f810bb1d 2024-11-27T12:24:58,524 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/.tmp/A/b2e8fe6209fb4574b62784fa5a292246 as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/A/b2e8fe6209fb4574b62784fa5a292246 2024-11-27T12:24:58,529 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/A/b2e8fe6209fb4574b62784fa5a292246, entries=250, sequenceid=309, filesize=47.5 K 2024-11-27T12:24:58,530 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/.tmp/B/1c0751f5da2148d7b03a0f6e6870dc43 as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/B/1c0751f5da2148d7b03a0f6e6870dc43 2024-11-27T12:24:58,535 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/B/1c0751f5da2148d7b03a0f6e6870dc43, entries=150, sequenceid=309, filesize=12.0 K 2024-11-27T12:24:58,536 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/.tmp/C/b83f65028cfc48628a10cec0f810bb1d as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/C/b83f65028cfc48628a10cec0f810bb1d 2024-11-27T12:24:58,543 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/C/b83f65028cfc48628a10cec0f810bb1d, entries=150, sequenceid=309, filesize=12.0 K 2024-11-27T12:24:58,546 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=134.18 KB/137400 for 5dcfd84eb369d011a0951d03c984f736 in 486ms, sequenceid=309, compaction requested=true 2024-11-27T12:24:58,546 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5dcfd84eb369d011a0951d03c984f736: 2024-11-27T12:24:58,546 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5dcfd84eb369d011a0951d03c984f736:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T12:24:58,546 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T12:24:58,546 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5dcfd84eb369d011a0951d03c984f736:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T12:24:58,546 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T12:24:58,546 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T12:24:58,546 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5dcfd84eb369d011a0951d03c984f736:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T12:24:58,546 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T12:24:58,546 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T12:24:58,547 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T12:24:58,547 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 111797 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T12:24:58,547 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HStore(1541): 5dcfd84eb369d011a0951d03c984f736/B is initiating minor compaction (all files) 2024-11-27T12:24:58,547 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HStore(1541): 5dcfd84eb369d011a0951d03c984f736/A is initiating minor compaction (all files) 2024-11-27T12:24:58,547 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5dcfd84eb369d011a0951d03c984f736/B in TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736. 2024-11-27T12:24:58,547 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5dcfd84eb369d011a0951d03c984f736/A in TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736. 2024-11-27T12:24:58,548 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/B/f3e7a2edbd234f9fa8eb1b0979229419, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/B/9786d1db24d145c8a2b9dbf40b7435f4, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/B/1c0751f5da2148d7b03a0f6e6870dc43] into tmpdir=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/.tmp, totalSize=36.7 K 2024-11-27T12:24:58,548 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/A/cef0ad5fa2554478b26473f6dd351323, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/A/a1b15381698b499a8023674f8cf9dd92, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/A/b2e8fe6209fb4574b62784fa5a292246] into tmpdir=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/.tmp, totalSize=109.2 K 2024-11-27T12:24:58,548 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736. 2024-11-27T12:24:58,548 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736. files: [hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/A/cef0ad5fa2554478b26473f6dd351323, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/A/a1b15381698b499a8023674f8cf9dd92, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/A/b2e8fe6209fb4574b62784fa5a292246] 2024-11-27T12:24:58,548 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.Compactor(225): Compacting f3e7a2edbd234f9fa8eb1b0979229419, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=267, earliestPutTs=1732710294096 2024-11-27T12:24:58,548 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] compactions.Compactor(225): Compacting cef0ad5fa2554478b26473f6dd351323, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=267, earliestPutTs=1732710294096 2024-11-27T12:24:58,548 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.Compactor(225): Compacting 9786d1db24d145c8a2b9dbf40b7435f4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=293, earliestPutTs=1732710295242 2024-11-27T12:24:58,548 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] compactions.Compactor(225): Compacting a1b15381698b499a8023674f8cf9dd92, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=293, earliestPutTs=1732710295242 2024-11-27T12:24:58,550 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.Compactor(225): Compacting 1c0751f5da2148d7b03a0f6e6870dc43, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=309, earliestPutTs=1732710297423 2024-11-27T12:24:58,550 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] compactions.Compactor(225): Compacting b2e8fe6209fb4574b62784fa5a292246, keycount=250, bloomtype=ROW, size=47.5 K, encoding=NONE, compression=NONE, seqNum=309, earliestPutTs=1732710297423 2024-11-27T12:24:58,557 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5dcfd84eb369d011a0951d03c984f736#B#compaction#165 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T12:24:58,558 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/.tmp/B/ea195e843366498ba86d6537bc7808d3 is 50, key is test_row_0/B:col10/1732710298059/Put/seqid=0 2024-11-27T12:24:58,559 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=5dcfd84eb369d011a0951d03c984f736] 2024-11-27T12:24:58,561 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742027_1203 (size=13051) 2024-11-27T12:24:58,562 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241127eb1c0692111d419695ba787f474bbc0b_5dcfd84eb369d011a0951d03c984f736 store=[table=TestAcidGuarantees family=A region=5dcfd84eb369d011a0951d03c984f736] 2024-11-27T12:24:58,564 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241127eb1c0692111d419695ba787f474bbc0b_5dcfd84eb369d011a0951d03c984f736, store=[table=TestAcidGuarantees family=A region=5dcfd84eb369d011a0951d03c984f736] 2024-11-27T12:24:58,564 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127eb1c0692111d419695ba787f474bbc0b_5dcfd84eb369d011a0951d03c984f736 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=5dcfd84eb369d011a0951d03c984f736] 2024-11-27T12:24:58,571 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742028_1204 (size=4469) 2024-11-27T12:24:58,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(8855): Flush requested on 5dcfd84eb369d011a0951d03c984f736 2024-11-27T12:24:58,761 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T12:24:58,761 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5dcfd84eb369d011a0951d03c984f736 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-27T12:24:58,775 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127f534c3798cf848fda669328c66d0a30f_5dcfd84eb369d011a0951d03c984f736 is 50, key is test_row_0/A:col10/1732710298759/Put/seqid=0 2024-11-27T12:24:58,779 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742029_1205 (size=14994) 2024-11-27T12:24:58,787 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:58,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44012 deadline: 1732710358783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:58,787 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:58,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44052 deadline: 1732710358783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:58,787 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:58,787 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:58,787 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:24:58,788 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:58,788 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:58,788 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:58,788 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:24:58,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44064 deadline: 1732710358784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:58,788 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:58,788 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:58,788 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:24:58,900 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:58,900 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:58,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44052 deadline: 1732710358898, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:58,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44012 deadline: 1732710358898, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:58,901 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:58,901 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:58,901 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:58,901 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:58,901 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:58,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44064 deadline: 1732710358898, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:58,901 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:24:58,901 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:24:58,901 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:58,901 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:58,901 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:24:58,967 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/.tmp/B/ea195e843366498ba86d6537bc7808d3 as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/B/ea195e843366498ba86d6537bc7808d3 2024-11-27T12:24:58,973 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5dcfd84eb369d011a0951d03c984f736#A#compaction#166 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T12:24:58,973 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5dcfd84eb369d011a0951d03c984f736/B of 5dcfd84eb369d011a0951d03c984f736 into ea195e843366498ba86d6537bc7808d3(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T12:24:58,973 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5dcfd84eb369d011a0951d03c984f736: 2024-11-27T12:24:58,973 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., storeName=5dcfd84eb369d011a0951d03c984f736/B, priority=13, startTime=1732710298546; duration=0sec 2024-11-27T12:24:58,973 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/.tmp/A/a84402acd09b4e0ba358d025f320590b is 175, key is test_row_0/A:col10/1732710298059/Put/seqid=0 2024-11-27T12:24:58,973 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T12:24:58,974 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5dcfd84eb369d011a0951d03c984f736:B 2024-11-27T12:24:58,974 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T12:24:58,976 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T12:24:58,976 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HStore(1541): 5dcfd84eb369d011a0951d03c984f736/C is initiating minor compaction (all files) 2024-11-27T12:24:58,976 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5dcfd84eb369d011a0951d03c984f736/C in TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736. 2024-11-27T12:24:58,976 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/C/2aa8553d58414fa8b3d7135027dc6059, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/C/2f4e973b26f94a1190a7d9a5321b9bcc, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/C/b83f65028cfc48628a10cec0f810bb1d] into tmpdir=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/.tmp, totalSize=36.7 K 2024-11-27T12:24:58,977 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.Compactor(225): Compacting 2aa8553d58414fa8b3d7135027dc6059, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=267, earliestPutTs=1732710294096 2024-11-27T12:24:58,977 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.Compactor(225): Compacting 2f4e973b26f94a1190a7d9a5321b9bcc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=293, earliestPutTs=1732710295242 2024-11-27T12:24:58,978 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.Compactor(225): Compacting b83f65028cfc48628a10cec0f810bb1d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=309, earliestPutTs=1732710297423 2024-11-27T12:24:58,994 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742030_1206 (size=32005) 2024-11-27T12:24:59,002 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5dcfd84eb369d011a0951d03c984f736#C#compaction#168 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T12:24:59,003 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/.tmp/C/ac8ddbf97dba452bb34307436b535cba is 50, key is test_row_0/C:col10/1732710298059/Put/seqid=0 2024-11-27T12:24:59,014 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742031_1207 (size=13051) 2024-11-27T12:24:59,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-27T12:24:59,098 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T12:24:59,098 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-27T12:24:59,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] procedure2.ProcedureExecutor(1139): Stored pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees 2024-11-27T12:24:59,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-27T12:24:59,101 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T12:24:59,101 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T12:24:59,101 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=51, ppid=50, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T12:24:59,109 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:59,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44052 deadline: 1732710359108, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:59,109 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:59,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44012 deadline: 1732710359108, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:59,109 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:59,109 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:59,109 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:24:59,109 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:59,109 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:59,109 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:59,109 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:24:59,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44064 deadline: 1732710359108, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:59,110 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:59,110 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:59,110 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:24:59,180 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:59,186 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127f534c3798cf848fda669328c66d0a30f_5dcfd84eb369d011a0951d03c984f736 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127f534c3798cf848fda669328c66d0a30f_5dcfd84eb369d011a0951d03c984f736 2024-11-27T12:24:59,187 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/.tmp/A/1904d91d46dc489ea204d1cb84bf96d7, store: [table=TestAcidGuarantees family=A region=5dcfd84eb369d011a0951d03c984f736] 2024-11-27T12:24:59,188 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/.tmp/A/1904d91d46dc489ea204d1cb84bf96d7 is 175, key is test_row_0/A:col10/1732710298759/Put/seqid=0 2024-11-27T12:24:59,194 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742032_1208 (size=39949) 2024-11-27T12:24:59,195 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=334, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/.tmp/A/1904d91d46dc489ea204d1cb84bf96d7 2024-11-27T12:24:59,207 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/.tmp/B/d25150077d074fdcbc29c511383de2db is 50, key is test_row_0/B:col10/1732710298759/Put/seqid=0 2024-11-27T12:24:59,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-27T12:24:59,224 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742033_1209 (size=12301) 2024-11-27T12:24:59,226 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=334 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/.tmp/B/d25150077d074fdcbc29c511383de2db 2024-11-27T12:24:59,237 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/.tmp/C/a4dd5a69204c41b88baf1ad69f09e63d is 50, key is test_row_0/C:col10/1732710298759/Put/seqid=0 2024-11-27T12:24:59,251 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742034_1210 (size=12301) 2024-11-27T12:24:59,255 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=334 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/.tmp/C/a4dd5a69204c41b88baf1ad69f09e63d 2024-11-27T12:24:59,255 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-27T12:24:59,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736. 2024-11-27T12:24:59,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736. as already flushing 2024-11-27T12:24:59,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736. 2024-11-27T12:24:59,256 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 5dcfd84eb369d011a0951d03c984f736, NAME => 'TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:24:59,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 5dcfd84eb369d011a0951d03c984f736, NAME => 'TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:24:59,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5dcfd84eb369d011a0951d03c984f736, NAME => 'TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5dcfd84eb369d011a0951d03c984f736, NAME => 'TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:24:59,262 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/.tmp/A/1904d91d46dc489ea204d1cb84bf96d7 as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/A/1904d91d46dc489ea204d1cb84bf96d7 2024-11-27T12:24:59,271 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/A/1904d91d46dc489ea204d1cb84bf96d7, entries=200, sequenceid=334, filesize=39.0 K 2024-11-27T12:24:59,275 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/.tmp/B/d25150077d074fdcbc29c511383de2db as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/B/d25150077d074fdcbc29c511383de2db 2024-11-27T12:24:59,279 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/B/d25150077d074fdcbc29c511383de2db, entries=150, sequenceid=334, filesize=12.0 K 2024-11-27T12:24:59,284 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/.tmp/C/a4dd5a69204c41b88baf1ad69f09e63d as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/C/a4dd5a69204c41b88baf1ad69f09e63d 2024-11-27T12:24:59,289 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/C/a4dd5a69204c41b88baf1ad69f09e63d, entries=150, sequenceid=334, filesize=12.0 K 2024-11-27T12:24:59,290 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 5dcfd84eb369d011a0951d03c984f736 in 529ms, sequenceid=334, compaction requested=false 2024-11-27T12:24:59,291 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5dcfd84eb369d011a0951d03c984f736: 2024-11-27T12:24:59,401 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/.tmp/A/a84402acd09b4e0ba358d025f320590b as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/A/a84402acd09b4e0ba358d025f320590b 2024-11-27T12:24:59,406 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5dcfd84eb369d011a0951d03c984f736/A of 5dcfd84eb369d011a0951d03c984f736 into a84402acd09b4e0ba358d025f320590b(size=31.3 K), total size for store is 70.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T12:24:59,406 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5dcfd84eb369d011a0951d03c984f736: 2024-11-27T12:24:59,406 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., storeName=5dcfd84eb369d011a0951d03c984f736/A, priority=13, startTime=1732710298546; duration=0sec 2024-11-27T12:24:59,407 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T12:24:59,407 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5dcfd84eb369d011a0951d03c984f736:A 2024-11-27T12:24:59,411 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-27T12:24:59,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736. 2024-11-27T12:24:59,412 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2902): Flushing 5dcfd84eb369d011a0951d03c984f736 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-27T12:24:59,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-27T12:24:59,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127c03360d845b04f4ea272059430d7ce55_5dcfd84eb369d011a0951d03c984f736 is 50, key is test_row_0/A:col10/1732710298780/Put/seqid=0 2024-11-27T12:24:59,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(8855): Flush requested on 5dcfd84eb369d011a0951d03c984f736 2024-11-27T12:24:59,421 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736. as already flushing 2024-11-27T12:24:59,426 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/.tmp/C/ac8ddbf97dba452bb34307436b535cba as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/C/ac8ddbf97dba452bb34307436b535cba 2024-11-27T12:24:59,430 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742035_1211 (size=12454) 2024-11-27T12:24:59,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:24:59,437 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127c03360d845b04f4ea272059430d7ce55_5dcfd84eb369d011a0951d03c984f736 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127c03360d845b04f4ea272059430d7ce55_5dcfd84eb369d011a0951d03c984f736 2024-11-27T12:24:59,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/.tmp/A/28e9b74667a947cfbb25386d3c2768f5, store: [table=TestAcidGuarantees family=A region=5dcfd84eb369d011a0951d03c984f736] 2024-11-27T12:24:59,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/.tmp/A/28e9b74667a947cfbb25386d3c2768f5 is 175, key is test_row_0/A:col10/1732710298780/Put/seqid=0 2024-11-27T12:24:59,442 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742036_1212 (size=31255) 2024-11-27T12:24:59,466 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5dcfd84eb369d011a0951d03c984f736/C of 5dcfd84eb369d011a0951d03c984f736 into ac8ddbf97dba452bb34307436b535cba(size=12.7 K), total size for store is 24.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T12:24:59,466 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5dcfd84eb369d011a0951d03c984f736: 2024-11-27T12:24:59,466 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., storeName=5dcfd84eb369d011a0951d03c984f736/C, priority=13, startTime=1732710298546; duration=0sec 2024-11-27T12:24:59,466 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T12:24:59,466 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5dcfd84eb369d011a0951d03c984f736:C 2024-11-27T12:24:59,503 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:59,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44012 deadline: 1732710359500, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:59,503 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:59,503 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:59,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44052 deadline: 1732710359500, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:59,503 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:59,503 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:24:59,503 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:59,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44064 deadline: 1732710359501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:59,504 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:59,504 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:59,504 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:24:59,504 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:59,504 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:59,504 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:24:59,612 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:59,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44012 deadline: 1732710359608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:59,613 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:59,613 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:59,613 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:59,613 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:59,613 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:24:59,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44064 deadline: 1732710359609, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:59,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44052 deadline: 1732710359609, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:59,613 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:59,613 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:59,613 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:59,613 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:24:59,613 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:59,613 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:24:59,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-27T12:24:59,819 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:59,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44052 deadline: 1732710359818, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:59,819 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:59,819 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:59,819 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:59,819 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:24:59,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44064 deadline: 1732710359818, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:59,819 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:24:59,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44012 deadline: 1732710359818, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:24:59,820 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:59,820 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:59,820 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:24:59,820 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:59,820 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:24:59,820 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:24:59,849 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=347, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/.tmp/A/28e9b74667a947cfbb25386d3c2768f5 2024-11-27T12:24:59,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/.tmp/B/a28ace894e484414a3ba9d1816495cfc is 50, key is test_row_0/B:col10/1732710298780/Put/seqid=0 2024-11-27T12:24:59,882 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742037_1213 (size=12301) 2024-11-27T12:24:59,888 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=347 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/.tmp/B/a28ace894e484414a3ba9d1816495cfc 2024-11-27T12:24:59,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/.tmp/C/bc6de40ac7a6408f95f1c0203e027d17 is 50, key is test_row_0/C:col10/1732710298780/Put/seqid=0 2024-11-27T12:24:59,914 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742038_1214 (size=12301) 2024-11-27T12:25:00,014 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-27T12:25:00,130 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:00,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44052 deadline: 1732710360128, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:00,131 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:00,131 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:00,131 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:00,131 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:25:00,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44064 deadline: 1732710360128, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:00,131 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:00,131 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:00,131 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:00,131 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:25:00,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44012 deadline: 1732710360128, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:00,131 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:00,131 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:00,131 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:25:00,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-27T12:25:00,315 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=347 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/.tmp/C/bc6de40ac7a6408f95f1c0203e027d17 2024-11-27T12:25:00,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/.tmp/A/28e9b74667a947cfbb25386d3c2768f5 as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/A/28e9b74667a947cfbb25386d3c2768f5 2024-11-27T12:25:00,324 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/A/28e9b74667a947cfbb25386d3c2768f5, entries=150, sequenceid=347, filesize=30.5 K 2024-11-27T12:25:00,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/.tmp/B/a28ace894e484414a3ba9d1816495cfc as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/B/a28ace894e484414a3ba9d1816495cfc 2024-11-27T12:25:00,339 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/B/a28ace894e484414a3ba9d1816495cfc, entries=150, sequenceid=347, filesize=12.0 K 2024-11-27T12:25:00,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/.tmp/C/bc6de40ac7a6408f95f1c0203e027d17 as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/C/bc6de40ac7a6408f95f1c0203e027d17 2024-11-27T12:25:00,346 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/C/bc6de40ac7a6408f95f1c0203e027d17, entries=150, sequenceid=347, filesize=12.0 K 2024-11-27T12:25:00,347 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 5dcfd84eb369d011a0951d03c984f736 in 934ms, sequenceid=347, compaction requested=true 2024-11-27T12:25:00,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2603): Flush status journal for 5dcfd84eb369d011a0951d03c984f736: 2024-11-27T12:25:00,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736. 2024-11-27T12:25:00,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=51 2024-11-27T12:25:00,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.HMaster(4169): Remote procedure done, pid=51 2024-11-27T12:25:00,350 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=51, resume processing ppid=50 2024-11-27T12:25:00,350 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=51, ppid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2470 sec 2024-11-27T12:25:00,351 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees in 1.2510 sec 2024-11-27T12:25:00,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(8855): Flush requested on 5dcfd84eb369d011a0951d03c984f736 2024-11-27T12:25:00,639 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T12:25:00,639 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5dcfd84eb369d011a0951d03c984f736 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-27T12:25:00,646 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:00,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44012 deadline: 1732710360644, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:00,647 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:00,647 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:00,647 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:25:00,647 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112740b20cea775f41d59b7e5f669fb20ba0_5dcfd84eb369d011a0951d03c984f736 is 50, key is test_row_0/A:col10/1732710300639/Put/seqid=0 2024-11-27T12:25:00,650 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:00,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44052 deadline: 1732710360645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:00,651 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:00,651 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:00,651 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:00,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44064 deadline: 1732710360646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:00,651 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:25:00,651 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:00,651 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:00,652 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:25:00,652 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742039_1215 (size=17534) 2024-11-27T12:25:00,762 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:00,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44012 deadline: 1732710360758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:00,762 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:00,762 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:00,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44052 deadline: 1732710360758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:00,762 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:00,762 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:25:00,762 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:00,762 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:00,762 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:00,762 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:25:00,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44064 deadline: 1732710360758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:00,763 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:00,763 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:00,763 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:25:00,968 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:00,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44012 deadline: 1732710360968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:00,969 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:00,969 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:00,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44052 deadline: 1732710360968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:00,969 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:00,969 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:25:00,969 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:00,969 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:00,969 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:00,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44064 deadline: 1732710360968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:00,969 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:25:00,969 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:00,969 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:00,969 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:25:01,053 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:25:01,057 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112740b20cea775f41d59b7e5f669fb20ba0_5dcfd84eb369d011a0951d03c984f736 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112740b20cea775f41d59b7e5f669fb20ba0_5dcfd84eb369d011a0951d03c984f736 2024-11-27T12:25:01,058 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/.tmp/A/00015685361f43a787e9b7cdc9719805, store: [table=TestAcidGuarantees family=A region=5dcfd84eb369d011a0951d03c984f736] 2024-11-27T12:25:01,058 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/.tmp/A/00015685361f43a787e9b7cdc9719805 is 175, key is test_row_0/A:col10/1732710300639/Put/seqid=0 2024-11-27T12:25:01,075 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742040_1216 (size=48639) 2024-11-27T12:25:01,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-27T12:25:01,259 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T12:25:01,259 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-27T12:25:01,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] procedure2.ProcedureExecutor(1139): Stored pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees 2024-11-27T12:25:01,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-27T12:25:01,261 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T12:25:01,262 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T12:25:01,262 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=53, ppid=52, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T12:25:01,278 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:01,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44052 deadline: 1732710361278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:01,279 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:01,279 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:01,279 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:01,279 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:25:01,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44012 deadline: 1732710361278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:01,279 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:01,279 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:01,279 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:25:01,280 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:01,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44064 deadline: 1732710361278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:01,281 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:01,281 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:01,281 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:25:01,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-27T12:25:01,414 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-27T12:25:01,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736. 2024-11-27T12:25:01,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736. as already flushing 2024-11-27T12:25:01,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736. 2024-11-27T12:25:01,415 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 5dcfd84eb369d011a0951d03c984f736, NAME => 'TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:25:01,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 5dcfd84eb369d011a0951d03c984f736, NAME => 'TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:25:01,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5dcfd84eb369d011a0951d03c984f736, NAME => 'TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5dcfd84eb369d011a0951d03c984f736, NAME => 'TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:25:01,476 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=375, memsize=53.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/.tmp/A/00015685361f43a787e9b7cdc9719805 2024-11-27T12:25:01,483 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/.tmp/B/1ee5455e5fb343868561858b095cc758 is 50, key is test_row_0/B:col10/1732710300639/Put/seqid=0 2024-11-27T12:25:01,487 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742041_1217 (size=12301) 2024-11-27T12:25:01,567 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-27T12:25:01,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736. 2024-11-27T12:25:01,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736. as already flushing 2024-11-27T12:25:01,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736. 2024-11-27T12:25:01,568 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 5dcfd84eb369d011a0951d03c984f736, NAME => 'TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:25:01,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 5dcfd84eb369d011a0951d03c984f736, NAME => 'TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:25:01,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5dcfd84eb369d011a0951d03c984f736, NAME => 'TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5dcfd84eb369d011a0951d03c984f736, NAME => 'TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:25:01,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-27T12:25:01,720 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-27T12:25:01,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736. 2024-11-27T12:25:01,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736. as already flushing 2024-11-27T12:25:01,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736. 2024-11-27T12:25:01,720 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 5dcfd84eb369d011a0951d03c984f736, NAME => 'TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:25:01,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 5dcfd84eb369d011a0951d03c984f736, NAME => 'TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:25:01,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5dcfd84eb369d011a0951d03c984f736, NAME => 'TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5dcfd84eb369d011a0951d03c984f736, NAME => 'TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:25:01,790 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:01,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44052 deadline: 1732710361788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:01,790 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:01,790 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:01,790 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:01,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44012 deadline: 1732710361788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:01,790 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:25:01,791 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:01,791 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:01,791 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:01,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44064 deadline: 1732710361788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:01,791 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:25:01,791 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:01,791 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:01,791 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:25:01,873 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-27T12:25:01,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736. 2024-11-27T12:25:01,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736. as already flushing 2024-11-27T12:25:01,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736. 2024-11-27T12:25:01,873 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 5dcfd84eb369d011a0951d03c984f736, NAME => 'TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:25:01,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 5dcfd84eb369d011a0951d03c984f736, NAME => 'TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:25:01,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5dcfd84eb369d011a0951d03c984f736, NAME => 'TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5dcfd84eb369d011a0951d03c984f736, NAME => 'TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:25:01,888 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=375 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/.tmp/B/1ee5455e5fb343868561858b095cc758 2024-11-27T12:25:01,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-27T12:25:01,895 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/.tmp/C/edb08f4747d1434aa15c385dba2d6f8b is 50, key is test_row_0/C:col10/1732710300639/Put/seqid=0 2024-11-27T12:25:01,899 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742042_1218 (size=12301) 2024-11-27T12:25:02,026 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-27T12:25:02,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736. 2024-11-27T12:25:02,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736. as already flushing 2024-11-27T12:25:02,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736. 2024-11-27T12:25:02,026 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 5dcfd84eb369d011a0951d03c984f736, NAME => 'TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:25:02,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 5dcfd84eb369d011a0951d03c984f736, NAME => 'TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:25:02,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5dcfd84eb369d011a0951d03c984f736, NAME => 'TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5dcfd84eb369d011a0951d03c984f736, NAME => 'TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:25:02,179 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-27T12:25:02,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736. 2024-11-27T12:25:02,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736. as already flushing 2024-11-27T12:25:02,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736. 2024-11-27T12:25:02,179 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 5dcfd84eb369d011a0951d03c984f736, NAME => 'TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:25:02,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 5dcfd84eb369d011a0951d03c984f736, NAME => 'TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:25:02,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5dcfd84eb369d011a0951d03c984f736, NAME => 'TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5dcfd84eb369d011a0951d03c984f736, NAME => 'TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:25:02,293 INFO [Thread-598 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-598. 2024-11-27T12:25:02,293 DEBUG [Thread-598 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T12:25:02,293 DEBUG [Thread-598 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T12:25:02,293 DEBUG [Thread-598 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T12:25:02,293 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T12:25:02,294 INFO [Thread-604 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-604. 2024-11-27T12:25:02,294 DEBUG [Thread-604 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T12:25:02,294 DEBUG [Thread-604 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T12:25:02,294 DEBUG [Thread-604 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T12:25:02,294 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T12:25:02,296 INFO [Thread-602 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-602. 2024-11-27T12:25:02,296 DEBUG [Thread-602 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T12:25:02,296 DEBUG [Thread-602 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T12:25:02,296 DEBUG [Thread-602 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T12:25:02,296 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T12:25:02,297 INFO [Thread-600 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-600. 2024-11-27T12:25:02,297 DEBUG [Thread-600 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T12:25:02,297 DEBUG [Thread-600 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T12:25:02,297 DEBUG [Thread-600 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T12:25:02,298 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T12:25:02,300 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=375 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/.tmp/C/edb08f4747d1434aa15c385dba2d6f8b 2024-11-27T12:25:02,306 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/.tmp/A/00015685361f43a787e9b7cdc9719805 as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/A/00015685361f43a787e9b7cdc9719805 2024-11-27T12:25:02,311 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/A/00015685361f43a787e9b7cdc9719805, entries=250, sequenceid=375, filesize=47.5 K 2024-11-27T12:25:02,312 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/.tmp/B/1ee5455e5fb343868561858b095cc758 as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/B/1ee5455e5fb343868561858b095cc758 2024-11-27T12:25:02,317 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/B/1ee5455e5fb343868561858b095cc758, entries=150, sequenceid=375, filesize=12.0 K 2024-11-27T12:25:02,318 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/.tmp/C/edb08f4747d1434aa15c385dba2d6f8b as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/C/edb08f4747d1434aa15c385dba2d6f8b 2024-11-27T12:25:02,319 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:02,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44010 deadline: 1732710362318, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:02,319 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to 1ba5af43fd99,36251,1732710241631 for 'test_row_2' in 5dcfd84eb369d011a0951d03c984f736 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8221 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T12:25:02,320 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:02,320 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:02,320 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:25:02,323 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/C/edb08f4747d1434aa15c385dba2d6f8b, entries=150, sequenceid=375, filesize=12.0 K 2024-11-27T12:25:02,324 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=40.25 KB/41220 for 5dcfd84eb369d011a0951d03c984f736 in 1685ms, sequenceid=375, compaction requested=true 2024-11-27T12:25:02,324 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5dcfd84eb369d011a0951d03c984f736: 2024-11-27T12:25:02,324 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5dcfd84eb369d011a0951d03c984f736:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T12:25:02,324 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-27T12:25:02,324 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T12:25:02,324 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5dcfd84eb369d011a0951d03c984f736:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T12:25:02,324 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T12:25:02,324 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-27T12:25:02,324 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5dcfd84eb369d011a0951d03c984f736:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T12:25:02,324 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T12:25:02,326 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49954 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-27T12:25:02,326 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 151848 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-27T12:25:02,326 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HStore(1541): 5dcfd84eb369d011a0951d03c984f736/A is initiating minor compaction (all files) 2024-11-27T12:25:02,326 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HStore(1541): 5dcfd84eb369d011a0951d03c984f736/B is initiating minor compaction (all files) 2024-11-27T12:25:02,326 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5dcfd84eb369d011a0951d03c984f736/B in TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736. 2024-11-27T12:25:02,326 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5dcfd84eb369d011a0951d03c984f736/A in TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736. 2024-11-27T12:25:02,326 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/B/ea195e843366498ba86d6537bc7808d3, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/B/d25150077d074fdcbc29c511383de2db, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/B/a28ace894e484414a3ba9d1816495cfc, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/B/1ee5455e5fb343868561858b095cc758] into tmpdir=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/.tmp, totalSize=48.8 K 2024-11-27T12:25:02,326 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/A/a84402acd09b4e0ba358d025f320590b, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/A/1904d91d46dc489ea204d1cb84bf96d7, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/A/28e9b74667a947cfbb25386d3c2768f5, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/A/00015685361f43a787e9b7cdc9719805] into tmpdir=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/.tmp, totalSize=148.3 K 2024-11-27T12:25:02,326 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736. 2024-11-27T12:25:02,326 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736. files: [hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/A/a84402acd09b4e0ba358d025f320590b, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/A/1904d91d46dc489ea204d1cb84bf96d7, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/A/28e9b74667a947cfbb25386d3c2768f5, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/A/00015685361f43a787e9b7cdc9719805] 2024-11-27T12:25:02,326 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.Compactor(225): Compacting ea195e843366498ba86d6537bc7808d3, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=309, earliestPutTs=1732710297423 2024-11-27T12:25:02,327 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] compactions.Compactor(225): Compacting a84402acd09b4e0ba358d025f320590b, keycount=150, bloomtype=ROW, size=31.3 K, encoding=NONE, compression=NONE, seqNum=309, earliestPutTs=1732710297423 2024-11-27T12:25:02,327 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.Compactor(225): Compacting d25150077d074fdcbc29c511383de2db, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=334, earliestPutTs=1732710298090 2024-11-27T12:25:02,327 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1904d91d46dc489ea204d1cb84bf96d7, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=334, earliestPutTs=1732710298090 2024-11-27T12:25:02,327 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.Compactor(225): Compacting a28ace894e484414a3ba9d1816495cfc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=347, earliestPutTs=1732710298776 2024-11-27T12:25:02,327 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] compactions.Compactor(225): Compacting 28e9b74667a947cfbb25386d3c2768f5, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=347, earliestPutTs=1732710298776 2024-11-27T12:25:02,327 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.Compactor(225): Compacting 1ee5455e5fb343868561858b095cc758, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=375, earliestPutTs=1732710299499 2024-11-27T12:25:02,328 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] compactions.Compactor(225): Compacting 00015685361f43a787e9b7cdc9719805, keycount=250, bloomtype=ROW, size=47.5 K, encoding=NONE, compression=NONE, seqNum=375, earliestPutTs=1732710299498 2024-11-27T12:25:02,329 INFO [Thread-593 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-593. 2024-11-27T12:25:02,329 DEBUG [Thread-593 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T12:25:02,329 DEBUG [Thread-593 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T12:25:02,330 DEBUG [Thread-593 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T12:25:02,330 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T12:25:02,333 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-27T12:25:02,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736. 2024-11-27T12:25:02,334 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2902): Flushing 5dcfd84eb369d011a0951d03c984f736 3/3 column families, dataSize=46.96 KB heapSize=123.80 KB 2024-11-27T12:25:02,339 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5dcfd84eb369d011a0951d03c984f736#B#compaction#177 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T12:25:02,340 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/.tmp/B/e9e8d1972245457da838fe10cc8f7f04 is 50, key is test_row_0/B:col10/1732710300639/Put/seqid=0 2024-11-27T12:25:02,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127a48369d5b4a640d1b9bf6fab65d4070a_5dcfd84eb369d011a0951d03c984f736 is 50, key is test_row_0/A:col10/1732710300642/Put/seqid=0 2024-11-27T12:25:02,351 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=5dcfd84eb369d011a0951d03c984f736] 2024-11-27T12:25:02,351 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742043_1219 (size=13187) 2024-11-27T12:25:02,359 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742044_1220 (size=12454) 2024-11-27T12:25:02,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:25:02,359 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241127813db3990a324ba1bb542aecd6be7f12_5dcfd84eb369d011a0951d03c984f736 store=[table=TestAcidGuarantees family=A region=5dcfd84eb369d011a0951d03c984f736] 2024-11-27T12:25:02,366 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127a48369d5b4a640d1b9bf6fab65d4070a_5dcfd84eb369d011a0951d03c984f736 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127a48369d5b4a640d1b9bf6fab65d4070a_5dcfd84eb369d011a0951d03c984f736 2024-11-27T12:25:02,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/.tmp/A/65c6471b30ef410caa6b54686c6245a5, store: [table=TestAcidGuarantees family=A region=5dcfd84eb369d011a0951d03c984f736] 2024-11-27T12:25:02,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/.tmp/A/65c6471b30ef410caa6b54686c6245a5 is 175, key is test_row_0/A:col10/1732710300642/Put/seqid=0 2024-11-27T12:25:02,374 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742045_1221 (size=31255) 2024-11-27T12:25:02,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-27T12:25:02,403 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241127813db3990a324ba1bb542aecd6be7f12_5dcfd84eb369d011a0951d03c984f736, store=[table=TestAcidGuarantees family=A region=5dcfd84eb369d011a0951d03c984f736] 2024-11-27T12:25:02,403 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127813db3990a324ba1bb542aecd6be7f12_5dcfd84eb369d011a0951d03c984f736 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=5dcfd84eb369d011a0951d03c984f736] 2024-11-27T12:25:02,407 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742046_1222 (size=4469) 2024-11-27T12:25:02,759 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/.tmp/B/e9e8d1972245457da838fe10cc8f7f04 as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/B/e9e8d1972245457da838fe10cc8f7f04 2024-11-27T12:25:02,766 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 5dcfd84eb369d011a0951d03c984f736/B of 5dcfd84eb369d011a0951d03c984f736 into e9e8d1972245457da838fe10cc8f7f04(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T12:25:02,766 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5dcfd84eb369d011a0951d03c984f736: 2024-11-27T12:25:02,766 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., storeName=5dcfd84eb369d011a0951d03c984f736/B, priority=12, startTime=1732710302324; duration=0sec 2024-11-27T12:25:02,767 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T12:25:02,767 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5dcfd84eb369d011a0951d03c984f736:B 2024-11-27T12:25:02,767 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-27T12:25:02,768 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49954 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-27T12:25:02,768 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HStore(1541): 5dcfd84eb369d011a0951d03c984f736/C is initiating minor compaction (all files) 2024-11-27T12:25:02,769 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5dcfd84eb369d011a0951d03c984f736/C in TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736. 2024-11-27T12:25:02,769 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/C/ac8ddbf97dba452bb34307436b535cba, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/C/a4dd5a69204c41b88baf1ad69f09e63d, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/C/bc6de40ac7a6408f95f1c0203e027d17, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/C/edb08f4747d1434aa15c385dba2d6f8b] into tmpdir=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/.tmp, totalSize=48.8 K 2024-11-27T12:25:02,769 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.Compactor(225): Compacting ac8ddbf97dba452bb34307436b535cba, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=309, earliestPutTs=1732710297423 2024-11-27T12:25:02,770 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.Compactor(225): Compacting a4dd5a69204c41b88baf1ad69f09e63d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=334, earliestPutTs=1732710298090 2024-11-27T12:25:02,770 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.Compactor(225): Compacting bc6de40ac7a6408f95f1c0203e027d17, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=347, earliestPutTs=1732710298776 2024-11-27T12:25:02,770 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.Compactor(225): Compacting edb08f4747d1434aa15c385dba2d6f8b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=375, earliestPutTs=1732710299499 2024-11-27T12:25:02,775 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=385, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/.tmp/A/65c6471b30ef410caa6b54686c6245a5 2024-11-27T12:25:02,780 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5dcfd84eb369d011a0951d03c984f736#C#compaction#180 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-27T12:25:02,781 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/.tmp/C/3dccfceda79347e0aec9d533c6b48fc9 is 50, key is test_row_0/C:col10/1732710300639/Put/seqid=0 2024-11-27T12:25:02,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/.tmp/B/7600347e7f4147eeb5eca3b4a2a9509c is 50, key is test_row_0/B:col10/1732710300642/Put/seqid=0 2024-11-27T12:25:02,785 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742047_1223 (size=13187) 2024-11-27T12:25:02,786 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742048_1224 (size=12301) 2024-11-27T12:25:02,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(8855): Flush requested on 5dcfd84eb369d011a0951d03c984f736 2024-11-27T12:25:02,799 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736. as already flushing 2024-11-27T12:25:02,799 INFO [Thread-589 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-589. 2024-11-27T12:25:02,799 DEBUG [Thread-589 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T12:25:02,799 DEBUG [Thread-589 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T12:25:02,799 INFO [Thread-595 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-595. 2024-11-27T12:25:02,799 DEBUG [Thread-595 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T12:25:02,799 DEBUG [Thread-589 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T12:25:02,799 DEBUG [Thread-595 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T12:25:02,799 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T12:25:02,799 DEBUG [Thread-595 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T12:25:02,799 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T12:25:02,808 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5dcfd84eb369d011a0951d03c984f736#A#compaction#179 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T12:25:02,809 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/.tmp/A/a0c459c3d49e4218947fee518dabb0d2 is 175, key is test_row_0/A:col10/1732710300639/Put/seqid=0 2024-11-27T12:25:02,809 INFO [Thread-591 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-591. 2024-11-27T12:25:02,810 DEBUG [Thread-591 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T12:25:02,810 DEBUG [Thread-591 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T12:25:02,810 DEBUG [Thread-591 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T12:25:02,810 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T12:25:02,813 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742049_1225 (size=32141) 2024-11-27T12:25:03,187 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=385 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/.tmp/B/7600347e7f4147eeb5eca3b4a2a9509c 2024-11-27T12:25:03,192 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/.tmp/C/3dccfceda79347e0aec9d533c6b48fc9 as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/C/3dccfceda79347e0aec9d533c6b48fc9 2024-11-27T12:25:03,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/.tmp/C/4a56e9173e7749f4b71b9f04c1639566 is 50, key is test_row_0/C:col10/1732710300642/Put/seqid=0 2024-11-27T12:25:03,199 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 5dcfd84eb369d011a0951d03c984f736/C of 5dcfd84eb369d011a0951d03c984f736 into 3dccfceda79347e0aec9d533c6b48fc9(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T12:25:03,199 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5dcfd84eb369d011a0951d03c984f736: 2024-11-27T12:25:03,199 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., storeName=5dcfd84eb369d011a0951d03c984f736/C, priority=12, startTime=1732710302324; duration=0sec 2024-11-27T12:25:03,199 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T12:25:03,199 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5dcfd84eb369d011a0951d03c984f736:C 2024-11-27T12:25:03,200 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742050_1226 (size=12301) 2024-11-27T12:25:03,218 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/.tmp/A/a0c459c3d49e4218947fee518dabb0d2 as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/A/a0c459c3d49e4218947fee518dabb0d2 2024-11-27T12:25:03,223 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 5dcfd84eb369d011a0951d03c984f736/A of 5dcfd84eb369d011a0951d03c984f736 into a0c459c3d49e4218947fee518dabb0d2(size=31.4 K), total size for store is 31.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T12:25:03,223 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5dcfd84eb369d011a0951d03c984f736: 2024-11-27T12:25:03,223 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., storeName=5dcfd84eb369d011a0951d03c984f736/A, priority=12, startTime=1732710302324; duration=0sec 2024-11-27T12:25:03,223 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T12:25:03,223 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5dcfd84eb369d011a0951d03c984f736:A 2024-11-27T12:25:03,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-27T12:25:03,603 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=385 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/.tmp/C/4a56e9173e7749f4b71b9f04c1639566 2024-11-27T12:25:03,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/.tmp/A/65c6471b30ef410caa6b54686c6245a5 as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/A/65c6471b30ef410caa6b54686c6245a5 2024-11-27T12:25:03,614 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/A/65c6471b30ef410caa6b54686c6245a5, entries=150, sequenceid=385, filesize=30.5 K 2024-11-27T12:25:03,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/.tmp/B/7600347e7f4147eeb5eca3b4a2a9509c as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/B/7600347e7f4147eeb5eca3b4a2a9509c 2024-11-27T12:25:03,619 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/B/7600347e7f4147eeb5eca3b4a2a9509c, entries=150, sequenceid=385, filesize=12.0 K 2024-11-27T12:25:03,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/.tmp/C/4a56e9173e7749f4b71b9f04c1639566 as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/C/4a56e9173e7749f4b71b9f04c1639566 2024-11-27T12:25:03,624 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/C/4a56e9173e7749f4b71b9f04c1639566, entries=150, sequenceid=385, filesize=12.0 K 2024-11-27T12:25:03,625 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~123.75 KB/126720, currentSize=20.13 KB/20610 for 5dcfd84eb369d011a0951d03c984f736 in 1291ms, sequenceid=385, compaction requested=false 2024-11-27T12:25:03,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2603): Flush status journal for 5dcfd84eb369d011a0951d03c984f736: 2024-11-27T12:25:03,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736. 2024-11-27T12:25:03,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=53 2024-11-27T12:25:03,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.HMaster(4169): Remote procedure done, pid=53 2024-11-27T12:25:03,629 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=53, resume processing ppid=52 2024-11-27T12:25:03,629 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=53, ppid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3650 sec 2024-11-27T12:25:03,631 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees in 2.3710 sec 2024-11-27T12:25:05,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-27T12:25:05,429 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T12:25:12,403 INFO [Thread-587 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-587. 2024-11-27T12:25:12,403 DEBUG [Thread-587 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T12:25:12,403 DEBUG [Thread-587 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T12:25:12,403 DEBUG [Thread-587 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T12:25:12,403 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T12:25:12,404 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-27T12:25:12,404 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 64 2024-11-27T12:25:12,405 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 57 2024-11-27T12:25:12,405 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 54 2024-11-27T12:25:12,405 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 71 2024-11-27T12:25:12,405 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 58 2024-11-27T12:25:12,405 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-27T12:25:12,405 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3543 2024-11-27T12:25:12,405 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3610 2024-11-27T12:25:12,405 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-27T12:25:12,405 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1544 2024-11-27T12:25:12,405 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4632 rows 2024-11-27T12:25:12,405 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1550 2024-11-27T12:25:12,405 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4650 rows 2024-11-27T12:25:12,405 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-27T12:25:12,406 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobMixedAtomicity(AcidGuaranteesTestBase.java:132) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-27T12:25:12,406 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T12:25:12,406 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T12:25:12,406 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T12:25:12,408 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-11-27T12:25:12,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] procedure2.ProcedureExecutor(1139): Stored pid=54, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-27T12:25:12,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-27T12:25:12,412 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732710312412"}]},"ts":"1732710312412"} 2024-11-27T12:25:12,414 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-27T12:25:12,415 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-27T12:25:12,416 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=55, ppid=54, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-27T12:25:12,417 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=56, ppid=55, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5dcfd84eb369d011a0951d03c984f736, UNASSIGN}] 2024-11-27T12:25:12,418 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=56, ppid=55, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5dcfd84eb369d011a0951d03c984f736, UNASSIGN 2024-11-27T12:25:12,420 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=56 updating hbase:meta row=5dcfd84eb369d011a0951d03c984f736, regionState=CLOSING, regionLocation=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:12,421 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=56, ppid=55, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=5dcfd84eb369d011a0951d03c984f736, UNASSIGN because future has completed 2024-11-27T12:25:12,422 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-27T12:25:12,422 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=57, ppid=56, state=RUNNABLE, hasLock=false; CloseRegionProcedure 5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631}] 2024-11-27T12:25:12,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-27T12:25:12,575 INFO [RS_CLOSE_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] handler.UnassignRegionHandler(122): Close 5dcfd84eb369d011a0951d03c984f736 2024-11-27T12:25:12,575 DEBUG [RS_CLOSE_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-27T12:25:12,575 DEBUG [RS_CLOSE_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(1722): Closing 5dcfd84eb369d011a0951d03c984f736, disabling compactions & flushes 2024-11-27T12:25:12,575 INFO [RS_CLOSE_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736. 2024-11-27T12:25:12,575 DEBUG [RS_CLOSE_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736. 2024-11-27T12:25:12,575 DEBUG [RS_CLOSE_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736. after waiting 0 ms 2024-11-27T12:25:12,575 DEBUG [RS_CLOSE_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736. 2024-11-27T12:25:12,575 INFO [RS_CLOSE_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(2902): Flushing 5dcfd84eb369d011a0951d03c984f736 3/3 column families, dataSize=26.84 KB heapSize=71.06 KB 2024-11-27T12:25:12,583 DEBUG [RS_CLOSE_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127acbb2fde6b994a789666840de8c72651_5dcfd84eb369d011a0951d03c984f736 is 50, key is test_row_0/A:col10/1732710302808/Put/seqid=0 2024-11-27T12:25:12,590 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742051_1227 (size=12454) 2024-11-27T12:25:12,590 DEBUG [RS_CLOSE_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:25:12,595 INFO [RS_CLOSE_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127acbb2fde6b994a789666840de8c72651_5dcfd84eb369d011a0951d03c984f736 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127acbb2fde6b994a789666840de8c72651_5dcfd84eb369d011a0951d03c984f736 2024-11-27T12:25:12,596 DEBUG [RS_CLOSE_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/.tmp/A/c07a4cc760b14645adfad3d8bff2d8fa, store: [table=TestAcidGuarantees family=A region=5dcfd84eb369d011a0951d03c984f736] 2024-11-27T12:25:12,596 DEBUG [RS_CLOSE_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/.tmp/A/c07a4cc760b14645adfad3d8bff2d8fa is 175, key is test_row_0/A:col10/1732710302808/Put/seqid=0 2024-11-27T12:25:12,600 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742052_1228 (size=31255) 2024-11-27T12:25:12,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-27T12:25:13,002 INFO [RS_CLOSE_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=395, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/.tmp/A/c07a4cc760b14645adfad3d8bff2d8fa 2024-11-27T12:25:13,016 DEBUG [RS_CLOSE_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/.tmp/B/60118e68a0144520a686e653560e85ba is 50, key is test_row_0/B:col10/1732710302808/Put/seqid=0 2024-11-27T12:25:13,020 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742053_1229 (size=12301) 2024-11-27T12:25:13,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-27T12:25:13,421 INFO [RS_CLOSE_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=395 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/.tmp/B/60118e68a0144520a686e653560e85ba 2024-11-27T12:25:13,428 DEBUG [RS_CLOSE_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/.tmp/C/b09e759322a04ed7a9e9186fa9e91c77 is 50, key is test_row_0/C:col10/1732710302808/Put/seqid=0 2024-11-27T12:25:13,433 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742054_1230 (size=12301) 2024-11-27T12:25:13,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-27T12:25:13,834 INFO [RS_CLOSE_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=395 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/.tmp/C/b09e759322a04ed7a9e9186fa9e91c77 2024-11-27T12:25:13,839 DEBUG [RS_CLOSE_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/.tmp/A/c07a4cc760b14645adfad3d8bff2d8fa as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/A/c07a4cc760b14645adfad3d8bff2d8fa 2024-11-27T12:25:13,844 INFO [RS_CLOSE_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/A/c07a4cc760b14645adfad3d8bff2d8fa, entries=150, sequenceid=395, filesize=30.5 K 2024-11-27T12:25:13,845 DEBUG [RS_CLOSE_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/.tmp/B/60118e68a0144520a686e653560e85ba as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/B/60118e68a0144520a686e653560e85ba 2024-11-27T12:25:13,850 INFO [RS_CLOSE_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/B/60118e68a0144520a686e653560e85ba, entries=150, sequenceid=395, filesize=12.0 K 2024-11-27T12:25:13,851 DEBUG [RS_CLOSE_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/.tmp/C/b09e759322a04ed7a9e9186fa9e91c77 as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/C/b09e759322a04ed7a9e9186fa9e91c77 2024-11-27T12:25:13,856 INFO [RS_CLOSE_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/C/b09e759322a04ed7a9e9186fa9e91c77, entries=150, sequenceid=395, filesize=12.0 K 2024-11-27T12:25:13,857 INFO [RS_CLOSE_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~71.02 KB/72720, currentSize=0 B/0 for 5dcfd84eb369d011a0951d03c984f736 in 1282ms, sequenceid=395, compaction requested=true 2024-11-27T12:25:13,858 DEBUG [StoreCloser-TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/A/087bb11cf4cb49c1b6f5f185314181d9, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/A/f96b021a7716418ca7c9c432ef18b536, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/A/414c1c6e7eb14998800c17b8a2c34935, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/A/46bb966630b2406ea1fd162932e6ffc3, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/A/db240c5207ea45f4bc89f2d332c1d35e, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/A/2d76e012974b4244a2e0c7b36db4bda5, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/A/3b6cbf421a1b4d08abf085c06e11befc, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/A/804bb279bfac454c9d73ce492a45e560, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/A/4343d343fba043e3b67f5f19dd5f4fbb, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/A/07caa68475a143369e2b52fa7f5e5acf, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/A/262b8f0fb1384bb19c60a861b52968d6, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/A/267bacd1d2ce460a873eb7cd2ac4eb3e, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/A/9a7a57745fe8463e8ad9140f48e99ee4, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/A/317c7fd53e3c42d5a0f2dee60904c9dc, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/A/22bf183acea34b48b397980f127c08f2, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/A/2d2e576b4abc4fceb474fa6c0f87aee5, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/A/976d62fcda1f4021a434eabaf16df647, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/A/da40a52557cf489391501eaeb20cba6b, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/A/712ff11a1ce344049497c0064807926f, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/A/cef0ad5fa2554478b26473f6dd351323, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/A/a1b15381698b499a8023674f8cf9dd92, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/A/b2e8fe6209fb4574b62784fa5a292246, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/A/a84402acd09b4e0ba358d025f320590b, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/A/1904d91d46dc489ea204d1cb84bf96d7, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/A/28e9b74667a947cfbb25386d3c2768f5, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/A/00015685361f43a787e9b7cdc9719805] to archive 2024-11-27T12:25:13,861 DEBUG [StoreCloser-TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-27T12:25:13,863 DEBUG [StoreCloser-TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/A/087bb11cf4cb49c1b6f5f185314181d9 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/A/087bb11cf4cb49c1b6f5f185314181d9 2024-11-27T12:25:13,864 DEBUG [StoreCloser-TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/A/f96b021a7716418ca7c9c432ef18b536 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/A/f96b021a7716418ca7c9c432ef18b536 2024-11-27T12:25:13,866 DEBUG [StoreCloser-TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/A/414c1c6e7eb14998800c17b8a2c34935 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/A/414c1c6e7eb14998800c17b8a2c34935 2024-11-27T12:25:13,868 DEBUG [StoreCloser-TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/A/46bb966630b2406ea1fd162932e6ffc3 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/A/46bb966630b2406ea1fd162932e6ffc3 2024-11-27T12:25:13,869 DEBUG [StoreCloser-TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/A/db240c5207ea45f4bc89f2d332c1d35e to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/A/db240c5207ea45f4bc89f2d332c1d35e 2024-11-27T12:25:13,873 DEBUG [StoreCloser-TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/A/2d76e012974b4244a2e0c7b36db4bda5 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/A/2d76e012974b4244a2e0c7b36db4bda5 2024-11-27T12:25:13,874 DEBUG [StoreCloser-TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/A/3b6cbf421a1b4d08abf085c06e11befc to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/A/3b6cbf421a1b4d08abf085c06e11befc 2024-11-27T12:25:13,876 DEBUG [StoreCloser-TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/A/804bb279bfac454c9d73ce492a45e560 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/A/804bb279bfac454c9d73ce492a45e560 2024-11-27T12:25:13,877 DEBUG [StoreCloser-TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/A/4343d343fba043e3b67f5f19dd5f4fbb to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/A/4343d343fba043e3b67f5f19dd5f4fbb 2024-11-27T12:25:13,879 DEBUG [StoreCloser-TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/A/07caa68475a143369e2b52fa7f5e5acf to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/A/07caa68475a143369e2b52fa7f5e5acf 2024-11-27T12:25:13,881 DEBUG [StoreCloser-TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/A/262b8f0fb1384bb19c60a861b52968d6 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/A/262b8f0fb1384bb19c60a861b52968d6 2024-11-27T12:25:13,882 DEBUG [StoreCloser-TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/A/267bacd1d2ce460a873eb7cd2ac4eb3e to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/A/267bacd1d2ce460a873eb7cd2ac4eb3e 2024-11-27T12:25:13,884 DEBUG [StoreCloser-TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/A/9a7a57745fe8463e8ad9140f48e99ee4 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/A/9a7a57745fe8463e8ad9140f48e99ee4 2024-11-27T12:25:13,886 DEBUG [StoreCloser-TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/A/317c7fd53e3c42d5a0f2dee60904c9dc to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/A/317c7fd53e3c42d5a0f2dee60904c9dc 2024-11-27T12:25:13,887 DEBUG [StoreCloser-TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/A/22bf183acea34b48b397980f127c08f2 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/A/22bf183acea34b48b397980f127c08f2 2024-11-27T12:25:13,888 DEBUG [StoreCloser-TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/A/2d2e576b4abc4fceb474fa6c0f87aee5 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/A/2d2e576b4abc4fceb474fa6c0f87aee5 2024-11-27T12:25:13,890 DEBUG [StoreCloser-TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/A/976d62fcda1f4021a434eabaf16df647 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/A/976d62fcda1f4021a434eabaf16df647 2024-11-27T12:25:13,892 DEBUG [StoreCloser-TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/A/da40a52557cf489391501eaeb20cba6b to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/A/da40a52557cf489391501eaeb20cba6b 2024-11-27T12:25:13,893 DEBUG [StoreCloser-TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/A/712ff11a1ce344049497c0064807926f to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/A/712ff11a1ce344049497c0064807926f 2024-11-27T12:25:13,895 DEBUG [StoreCloser-TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/A/cef0ad5fa2554478b26473f6dd351323 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/A/cef0ad5fa2554478b26473f6dd351323 2024-11-27T12:25:13,896 DEBUG [StoreCloser-TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/A/a1b15381698b499a8023674f8cf9dd92 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/A/a1b15381698b499a8023674f8cf9dd92 2024-11-27T12:25:13,897 DEBUG [StoreCloser-TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/A/b2e8fe6209fb4574b62784fa5a292246 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/A/b2e8fe6209fb4574b62784fa5a292246 2024-11-27T12:25:13,898 DEBUG [StoreCloser-TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/A/a84402acd09b4e0ba358d025f320590b to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/A/a84402acd09b4e0ba358d025f320590b 2024-11-27T12:25:13,900 DEBUG [StoreCloser-TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/A/1904d91d46dc489ea204d1cb84bf96d7 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/A/1904d91d46dc489ea204d1cb84bf96d7 2024-11-27T12:25:13,901 DEBUG [StoreCloser-TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/A/28e9b74667a947cfbb25386d3c2768f5 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/A/28e9b74667a947cfbb25386d3c2768f5 2024-11-27T12:25:13,902 DEBUG [StoreCloser-TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/A/00015685361f43a787e9b7cdc9719805 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/A/00015685361f43a787e9b7cdc9719805 2024-11-27T12:25:13,904 DEBUG [StoreCloser-TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/B/583ff06821a042adba3819a0e06cc3e4, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/B/c3cc05cc3520420491c4ce2d8926e957, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/B/5b7cf336c8f84d09921b6fd62da7b25f, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/B/0a7033c1fee349d688893b3bd4a6f02d, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/B/0555cd7548fe4505a89fd66d47434099, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/B/722bb37ed1bb491bb719a577c007ff73, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/B/b14ac58ab7454b569b4e1c12c06918bf, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/B/d0eebbcdf107487e9c5d5839048de3f0, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/B/ef23e15caac742c896111a802b9214d4, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/B/bb60aa00797249d1afc33648f68b319c, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/B/1effae4977224fea96e128ba90c2c146, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/B/2d24a83b91ee40dbab6ea579fd6eb9f2, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/B/1d6635e9fbaf47b28517ad2324e0f016, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/B/034f8578fbab48268a0c473d3c8ada72, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/B/c2b20b339ea344e8873f7c5ab9c9e309, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/B/bd3c6eabab244e61a13f032655ce498e, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/B/c2b805983df84a24ade534ee171115d1, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/B/afdf4e6b2a0e46f1b51d52418f9df0c6, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/B/f3e7a2edbd234f9fa8eb1b0979229419, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/B/1a7df16e395b48378b208512675e8475, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/B/9786d1db24d145c8a2b9dbf40b7435f4, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/B/ea195e843366498ba86d6537bc7808d3, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/B/1c0751f5da2148d7b03a0f6e6870dc43, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/B/d25150077d074fdcbc29c511383de2db, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/B/a28ace894e484414a3ba9d1816495cfc, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/B/1ee5455e5fb343868561858b095cc758] to archive 2024-11-27T12:25:13,905 DEBUG [StoreCloser-TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-27T12:25:13,908 DEBUG [StoreCloser-TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/B/583ff06821a042adba3819a0e06cc3e4 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/B/583ff06821a042adba3819a0e06cc3e4 2024-11-27T12:25:13,910 DEBUG [StoreCloser-TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/B/c3cc05cc3520420491c4ce2d8926e957 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/B/c3cc05cc3520420491c4ce2d8926e957 2024-11-27T12:25:13,912 DEBUG [StoreCloser-TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/B/5b7cf336c8f84d09921b6fd62da7b25f to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/B/5b7cf336c8f84d09921b6fd62da7b25f 2024-11-27T12:25:13,914 DEBUG [StoreCloser-TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/B/0a7033c1fee349d688893b3bd4a6f02d to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/B/0a7033c1fee349d688893b3bd4a6f02d 2024-11-27T12:25:13,915 DEBUG [StoreCloser-TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/B/0555cd7548fe4505a89fd66d47434099 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/B/0555cd7548fe4505a89fd66d47434099 2024-11-27T12:25:13,917 DEBUG [StoreCloser-TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/B/722bb37ed1bb491bb719a577c007ff73 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/B/722bb37ed1bb491bb719a577c007ff73 2024-11-27T12:25:13,919 DEBUG [StoreCloser-TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/B/b14ac58ab7454b569b4e1c12c06918bf to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/B/b14ac58ab7454b569b4e1c12c06918bf 2024-11-27T12:25:13,921 DEBUG [StoreCloser-TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/B/d0eebbcdf107487e9c5d5839048de3f0 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/B/d0eebbcdf107487e9c5d5839048de3f0 2024-11-27T12:25:13,922 DEBUG [StoreCloser-TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/B/ef23e15caac742c896111a802b9214d4 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/B/ef23e15caac742c896111a802b9214d4 2024-11-27T12:25:13,924 DEBUG [StoreCloser-TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/B/bb60aa00797249d1afc33648f68b319c to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/B/bb60aa00797249d1afc33648f68b319c 2024-11-27T12:25:13,925 DEBUG [StoreCloser-TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/B/1effae4977224fea96e128ba90c2c146 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/B/1effae4977224fea96e128ba90c2c146 2024-11-27T12:25:13,927 DEBUG [StoreCloser-TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/B/2d24a83b91ee40dbab6ea579fd6eb9f2 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/B/2d24a83b91ee40dbab6ea579fd6eb9f2 2024-11-27T12:25:13,928 DEBUG [StoreCloser-TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/B/1d6635e9fbaf47b28517ad2324e0f016 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/B/1d6635e9fbaf47b28517ad2324e0f016 2024-11-27T12:25:13,930 DEBUG [StoreCloser-TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/B/034f8578fbab48268a0c473d3c8ada72 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/B/034f8578fbab48268a0c473d3c8ada72 2024-11-27T12:25:13,932 DEBUG [StoreCloser-TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/B/c2b20b339ea344e8873f7c5ab9c9e309 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/B/c2b20b339ea344e8873f7c5ab9c9e309 2024-11-27T12:25:13,933 DEBUG [StoreCloser-TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/B/bd3c6eabab244e61a13f032655ce498e to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/B/bd3c6eabab244e61a13f032655ce498e 2024-11-27T12:25:13,934 DEBUG [StoreCloser-TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/B/c2b805983df84a24ade534ee171115d1 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/B/c2b805983df84a24ade534ee171115d1 2024-11-27T12:25:13,936 DEBUG [StoreCloser-TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/B/afdf4e6b2a0e46f1b51d52418f9df0c6 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/B/afdf4e6b2a0e46f1b51d52418f9df0c6 2024-11-27T12:25:13,937 DEBUG [StoreCloser-TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/B/f3e7a2edbd234f9fa8eb1b0979229419 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/B/f3e7a2edbd234f9fa8eb1b0979229419 2024-11-27T12:25:13,939 DEBUG [StoreCloser-TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/B/1a7df16e395b48378b208512675e8475 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/B/1a7df16e395b48378b208512675e8475 2024-11-27T12:25:13,942 DEBUG [StoreCloser-TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/B/9786d1db24d145c8a2b9dbf40b7435f4 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/B/9786d1db24d145c8a2b9dbf40b7435f4 2024-11-27T12:25:13,943 DEBUG [StoreCloser-TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/B/ea195e843366498ba86d6537bc7808d3 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/B/ea195e843366498ba86d6537bc7808d3 2024-11-27T12:25:13,945 DEBUG [StoreCloser-TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/B/1c0751f5da2148d7b03a0f6e6870dc43 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/B/1c0751f5da2148d7b03a0f6e6870dc43 2024-11-27T12:25:13,946 DEBUG [StoreCloser-TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/B/d25150077d074fdcbc29c511383de2db to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/B/d25150077d074fdcbc29c511383de2db 2024-11-27T12:25:13,948 DEBUG [StoreCloser-TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/B/a28ace894e484414a3ba9d1816495cfc to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/B/a28ace894e484414a3ba9d1816495cfc 2024-11-27T12:25:13,949 DEBUG [StoreCloser-TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/B/1ee5455e5fb343868561858b095cc758 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/B/1ee5455e5fb343868561858b095cc758 2024-11-27T12:25:13,956 DEBUG [StoreCloser-TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/C/15779297379e4155b260778392f26a78, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/C/22e85f9daff94ca59a9ae29b0df314d9, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/C/5cd61286223d4c38b7803121392d3c35, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/C/39b7bd0de2a94d18af17d35542f12ede, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/C/ddf504151c0f44f4ad64747ce1981b5d, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/C/c72219f77d354bc1b75f4ebde7cff5bb, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/C/003f0c162c9b4dc4b204b106a34c8c90, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/C/53c96489235d4ef1b96340d3b844a0d3, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/C/5034c2ba00694a2fb3caa87aa991bac5, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/C/7b0367c24548407389576730d8bc4254, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/C/1a7819d8f9424fe38ab206b97a858095, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/C/901883a452494d61a2c7930a51105060, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/C/c0deea0276bd439c9c8ee7bf44c7a2e7, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/C/331b562357eb46e8aab3dc87e8b61d65, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/C/5663d761d6cc43b8be1a74e5d690723d, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/C/0bbb910c50294a0d9629ddcb403a33e7, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/C/a1a5a39a52f84dfebd19cfadb79a7fb8, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/C/99394b71d8bd45d3a7914292b67e306c, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/C/2aa8553d58414fa8b3d7135027dc6059, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/C/fb1b2473ef854d8fb92516fb1650f5b6, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/C/2f4e973b26f94a1190a7d9a5321b9bcc, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/C/ac8ddbf97dba452bb34307436b535cba, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/C/b83f65028cfc48628a10cec0f810bb1d, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/C/a4dd5a69204c41b88baf1ad69f09e63d, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/C/bc6de40ac7a6408f95f1c0203e027d17, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/C/edb08f4747d1434aa15c385dba2d6f8b] to archive 2024-11-27T12:25:13,958 DEBUG [StoreCloser-TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-27T12:25:13,960 DEBUG [StoreCloser-TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/C/15779297379e4155b260778392f26a78 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/C/15779297379e4155b260778392f26a78 2024-11-27T12:25:13,961 DEBUG [StoreCloser-TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/C/22e85f9daff94ca59a9ae29b0df314d9 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/C/22e85f9daff94ca59a9ae29b0df314d9 2024-11-27T12:25:13,963 DEBUG [StoreCloser-TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/C/5cd61286223d4c38b7803121392d3c35 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/C/5cd61286223d4c38b7803121392d3c35 2024-11-27T12:25:13,964 DEBUG [StoreCloser-TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/C/39b7bd0de2a94d18af17d35542f12ede to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/C/39b7bd0de2a94d18af17d35542f12ede 2024-11-27T12:25:13,965 DEBUG [StoreCloser-TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/C/ddf504151c0f44f4ad64747ce1981b5d to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/C/ddf504151c0f44f4ad64747ce1981b5d 2024-11-27T12:25:13,967 DEBUG [StoreCloser-TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/C/c72219f77d354bc1b75f4ebde7cff5bb to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/C/c72219f77d354bc1b75f4ebde7cff5bb 2024-11-27T12:25:13,968 DEBUG [StoreCloser-TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/C/003f0c162c9b4dc4b204b106a34c8c90 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/C/003f0c162c9b4dc4b204b106a34c8c90 2024-11-27T12:25:13,969 DEBUG [StoreCloser-TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/C/53c96489235d4ef1b96340d3b844a0d3 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/C/53c96489235d4ef1b96340d3b844a0d3 2024-11-27T12:25:13,970 DEBUG [StoreCloser-TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/C/5034c2ba00694a2fb3caa87aa991bac5 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/C/5034c2ba00694a2fb3caa87aa991bac5 2024-11-27T12:25:13,972 DEBUG [StoreCloser-TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/C/7b0367c24548407389576730d8bc4254 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/C/7b0367c24548407389576730d8bc4254 2024-11-27T12:25:13,973 DEBUG [StoreCloser-TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/C/1a7819d8f9424fe38ab206b97a858095 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/C/1a7819d8f9424fe38ab206b97a858095 2024-11-27T12:25:13,974 DEBUG [StoreCloser-TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/C/901883a452494d61a2c7930a51105060 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/C/901883a452494d61a2c7930a51105060 2024-11-27T12:25:13,975 DEBUG [StoreCloser-TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/C/c0deea0276bd439c9c8ee7bf44c7a2e7 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/C/c0deea0276bd439c9c8ee7bf44c7a2e7 2024-11-27T12:25:13,977 DEBUG [StoreCloser-TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/C/331b562357eb46e8aab3dc87e8b61d65 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/C/331b562357eb46e8aab3dc87e8b61d65 2024-11-27T12:25:13,978 DEBUG [StoreCloser-TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/C/5663d761d6cc43b8be1a74e5d690723d to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/C/5663d761d6cc43b8be1a74e5d690723d 2024-11-27T12:25:13,979 DEBUG [StoreCloser-TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/C/0bbb910c50294a0d9629ddcb403a33e7 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/C/0bbb910c50294a0d9629ddcb403a33e7 2024-11-27T12:25:13,980 DEBUG [StoreCloser-TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/C/a1a5a39a52f84dfebd19cfadb79a7fb8 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/C/a1a5a39a52f84dfebd19cfadb79a7fb8 2024-11-27T12:25:13,981 DEBUG [StoreCloser-TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/C/99394b71d8bd45d3a7914292b67e306c to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/C/99394b71d8bd45d3a7914292b67e306c 2024-11-27T12:25:13,982 DEBUG [StoreCloser-TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/C/2aa8553d58414fa8b3d7135027dc6059 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/C/2aa8553d58414fa8b3d7135027dc6059 2024-11-27T12:25:13,983 DEBUG [StoreCloser-TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/C/fb1b2473ef854d8fb92516fb1650f5b6 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/C/fb1b2473ef854d8fb92516fb1650f5b6 2024-11-27T12:25:13,986 DEBUG [StoreCloser-TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/C/2f4e973b26f94a1190a7d9a5321b9bcc to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/C/2f4e973b26f94a1190a7d9a5321b9bcc 2024-11-27T12:25:13,987 DEBUG [StoreCloser-TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/C/ac8ddbf97dba452bb34307436b535cba to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/C/ac8ddbf97dba452bb34307436b535cba 2024-11-27T12:25:13,994 DEBUG [StoreCloser-TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/C/b83f65028cfc48628a10cec0f810bb1d to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/C/b83f65028cfc48628a10cec0f810bb1d 2024-11-27T12:25:13,997 DEBUG [StoreCloser-TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/C/a4dd5a69204c41b88baf1ad69f09e63d to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/C/a4dd5a69204c41b88baf1ad69f09e63d 2024-11-27T12:25:13,999 DEBUG [StoreCloser-TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/C/bc6de40ac7a6408f95f1c0203e027d17 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/C/bc6de40ac7a6408f95f1c0203e027d17 2024-11-27T12:25:14,003 DEBUG [StoreCloser-TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/C/edb08f4747d1434aa15c385dba2d6f8b to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/C/edb08f4747d1434aa15c385dba2d6f8b 2024-11-27T12:25:14,013 DEBUG [RS_CLOSE_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/recovered.edits/398.seqid, newMaxSeqId=398, maxSeqId=4 2024-11-27T12:25:14,013 INFO [RS_CLOSE_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736. 2024-11-27T12:25:14,014 DEBUG [RS_CLOSE_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(1676): Region close journal for 5dcfd84eb369d011a0951d03c984f736: Waiting for close lock at 1732710312575Running coprocessor pre-close hooks at 1732710312575Disabling compacts and flushes for region at 1732710312575Disabling writes for close at 1732710312575Obtaining lock to block concurrent updates at 1732710312575Preparing flush snapshotting stores in 5dcfd84eb369d011a0951d03c984f736 at 1732710312575Finished memstore snapshotting TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=72720, getOffHeapSize=0, getCellsCount=600 at 1732710312576 (+1 ms)Flushing stores of TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736. at 1732710312576Flushing 5dcfd84eb369d011a0951d03c984f736/A: creating writer at 1732710312576Flushing mob file 5dcfd84eb369d011a0951d03c984f736/A: appending metadata at 1732710312583 (+7 ms)Flushing mob file 5dcfd84eb369d011a0951d03c984f736/A: closing flushed file at 1732710312583Flushing 5dcfd84eb369d011a0951d03c984f736/A: appending metadata at 1732710312596 (+13 ms)Flushing 5dcfd84eb369d011a0951d03c984f736/A: closing flushed file at 1732710312596Flushing 5dcfd84eb369d011a0951d03c984f736/B: creating writer at 1732710313011 (+415 ms)Flushing 5dcfd84eb369d011a0951d03c984f736/B: appending metadata at 1732710313016 (+5 ms)Flushing 5dcfd84eb369d011a0951d03c984f736/B: closing flushed file at 1732710313016Flushing 5dcfd84eb369d011a0951d03c984f736/C: creating writer at 1732710313425 (+409 ms)Flushing 5dcfd84eb369d011a0951d03c984f736/C: appending metadata at 1732710313428 (+3 ms)Flushing 5dcfd84eb369d011a0951d03c984f736/C: closing flushed file at 1732710313428Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5d3d32b: reopening flushed file at 1732710313838 (+410 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@616126ba: reopening flushed file at 1732710313844 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3d9517c9: reopening flushed file at 1732710313850 (+6 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~71.02 KB/72720, currentSize=0 B/0 for 5dcfd84eb369d011a0951d03c984f736 in 1282ms, sequenceid=395, compaction requested=true at 1732710313857 (+7 ms)Writing region close event to WAL at 1732710314005 (+148 ms)Running coprocessor post-close hooks at 1732710314013 (+8 ms)Closed at 1732710314013 2024-11-27T12:25:14,019 INFO [RS_CLOSE_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] handler.UnassignRegionHandler(157): Closed 5dcfd84eb369d011a0951d03c984f736 2024-11-27T12:25:14,019 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=56 updating hbase:meta row=5dcfd84eb369d011a0951d03c984f736, regionState=CLOSED 2024-11-27T12:25:14,021 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=57, ppid=56, state=RUNNABLE, hasLock=false; CloseRegionProcedure 5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 because future has completed 2024-11-27T12:25:14,023 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=57, resume processing ppid=56 2024-11-27T12:25:14,024 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=57, ppid=56, state=SUCCESS, hasLock=false; CloseRegionProcedure 5dcfd84eb369d011a0951d03c984f736, server=1ba5af43fd99,36251,1732710241631 in 1.6000 sec 2024-11-27T12:25:14,025 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=56, resume processing ppid=55 2024-11-27T12:25:14,025 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=56, ppid=55, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5dcfd84eb369d011a0951d03c984f736, UNASSIGN in 1.6070 sec 2024-11-27T12:25:14,028 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=55, resume processing ppid=54 2024-11-27T12:25:14,029 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=55, ppid=54, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.6110 sec 2024-11-27T12:25:14,030 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732710314030"}]},"ts":"1732710314030"} 2024-11-27T12:25:14,032 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-27T12:25:14,032 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-27T12:25:14,039 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=54, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.6250 sec 2024-11-27T12:25:14,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-27T12:25:14,559 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-27T12:25:14,559 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-11-27T12:25:14,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] procedure2.ProcedureExecutor(1139): Stored pid=58, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-27T12:25:14,561 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=58, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-27T12:25:14,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-27T12:25:14,562 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=58, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-27T12:25:14,566 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736 2024-11-27T12:25:14,569 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/A, FileablePath, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/B, FileablePath, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/C, FileablePath, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/recovered.edits] 2024-11-27T12:25:14,572 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/A/65c6471b30ef410caa6b54686c6245a5 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/A/65c6471b30ef410caa6b54686c6245a5 2024-11-27T12:25:14,573 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/A/a0c459c3d49e4218947fee518dabb0d2 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/A/a0c459c3d49e4218947fee518dabb0d2 2024-11-27T12:25:14,575 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/A/c07a4cc760b14645adfad3d8bff2d8fa to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/A/c07a4cc760b14645adfad3d8bff2d8fa 2024-11-27T12:25:14,578 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/B/60118e68a0144520a686e653560e85ba to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/B/60118e68a0144520a686e653560e85ba 2024-11-27T12:25:14,579 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/B/7600347e7f4147eeb5eca3b4a2a9509c to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/B/7600347e7f4147eeb5eca3b4a2a9509c 2024-11-27T12:25:14,580 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/B/e9e8d1972245457da838fe10cc8f7f04 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/B/e9e8d1972245457da838fe10cc8f7f04 2024-11-27T12:25:14,582 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/C/3dccfceda79347e0aec9d533c6b48fc9 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/C/3dccfceda79347e0aec9d533c6b48fc9 2024-11-27T12:25:14,584 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/C/4a56e9173e7749f4b71b9f04c1639566 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/C/4a56e9173e7749f4b71b9f04c1639566 2024-11-27T12:25:14,585 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/C/b09e759322a04ed7a9e9186fa9e91c77 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/C/b09e759322a04ed7a9e9186fa9e91c77 2024-11-27T12:25:14,588 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/recovered.edits/398.seqid to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736/recovered.edits/398.seqid 2024-11-27T12:25:14,589 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/5dcfd84eb369d011a0951d03c984f736 2024-11-27T12:25:14,589 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-27T12:25:14,589 DEBUG [PEWorker-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-27T12:25:14,590 DEBUG [PEWorker-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-27T12:25:14,595 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411272715c88339fe4f3c8b8a2526d98f63ad_5dcfd84eb369d011a0951d03c984f736 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411272715c88339fe4f3c8b8a2526d98f63ad_5dcfd84eb369d011a0951d03c984f736 2024-11-27T12:25:14,596 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411272dd5ebe1ef434bcabdcee8ee3f7298dc_5dcfd84eb369d011a0951d03c984f736 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411272dd5ebe1ef434bcabdcee8ee3f7298dc_5dcfd84eb369d011a0951d03c984f736 2024-11-27T12:25:14,597 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112738e2bb034fe944338de9d02abdb8e5ea_5dcfd84eb369d011a0951d03c984f736 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112738e2bb034fe944338de9d02abdb8e5ea_5dcfd84eb369d011a0951d03c984f736 2024-11-27T12:25:14,599 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411273cb2605994f24c41a2f66534a5acedf7_5dcfd84eb369d011a0951d03c984f736 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411273cb2605994f24c41a2f66534a5acedf7_5dcfd84eb369d011a0951d03c984f736 2024-11-27T12:25:14,600 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112740b20cea775f41d59b7e5f669fb20ba0_5dcfd84eb369d011a0951d03c984f736 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112740b20cea775f41d59b7e5f669fb20ba0_5dcfd84eb369d011a0951d03c984f736 2024-11-27T12:25:14,602 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127451a9c6fad334d63bcf8750b0c271ee2_5dcfd84eb369d011a0951d03c984f736 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127451a9c6fad334d63bcf8750b0c271ee2_5dcfd84eb369d011a0951d03c984f736 2024-11-27T12:25:14,603 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411274cdfccf735de4d50a4bda8ac37409551_5dcfd84eb369d011a0951d03c984f736 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411274cdfccf735de4d50a4bda8ac37409551_5dcfd84eb369d011a0951d03c984f736 2024-11-27T12:25:14,604 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411275317aa3618b14c1cb6c6e549430d1a2b_5dcfd84eb369d011a0951d03c984f736 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411275317aa3618b14c1cb6c6e549430d1a2b_5dcfd84eb369d011a0951d03c984f736 2024-11-27T12:25:14,606 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411275db2e51773a6487db7cf3339316a061b_5dcfd84eb369d011a0951d03c984f736 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411275db2e51773a6487db7cf3339316a061b_5dcfd84eb369d011a0951d03c984f736 2024-11-27T12:25:14,607 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411277c13fda6fa4f44fe9504df085ddc48a2_5dcfd84eb369d011a0951d03c984f736 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411277c13fda6fa4f44fe9504df085ddc48a2_5dcfd84eb369d011a0951d03c984f736 2024-11-27T12:25:14,608 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411277d5a9ee263bc4296aabd2b994c5bd397_5dcfd84eb369d011a0951d03c984f736 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411277d5a9ee263bc4296aabd2b994c5bd397_5dcfd84eb369d011a0951d03c984f736 2024-11-27T12:25:14,610 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411277e6cb2181a244bf0bd5081cca9d3dda1_5dcfd84eb369d011a0951d03c984f736 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411277e6cb2181a244bf0bd5081cca9d3dda1_5dcfd84eb369d011a0951d03c984f736 2024-11-27T12:25:14,610 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127852cec6eede742e7a5018b6919bc5316_5dcfd84eb369d011a0951d03c984f736 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127852cec6eede742e7a5018b6919bc5316_5dcfd84eb369d011a0951d03c984f736 2024-11-27T12:25:14,611 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127892323c757d84c68b1b888bb13a592cb_5dcfd84eb369d011a0951d03c984f736 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127892323c757d84c68b1b888bb13a592cb_5dcfd84eb369d011a0951d03c984f736 2024-11-27T12:25:14,613 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411279478eb62e8bd46a2a95c36d4895ed1bc_5dcfd84eb369d011a0951d03c984f736 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411279478eb62e8bd46a2a95c36d4895ed1bc_5dcfd84eb369d011a0951d03c984f736 2024-11-27T12:25:14,615 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127a1b5d566004a4b3fa802ed2d08bd677a_5dcfd84eb369d011a0951d03c984f736 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127a1b5d566004a4b3fa802ed2d08bd677a_5dcfd84eb369d011a0951d03c984f736 2024-11-27T12:25:14,616 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127a48369d5b4a640d1b9bf6fab65d4070a_5dcfd84eb369d011a0951d03c984f736 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127a48369d5b4a640d1b9bf6fab65d4070a_5dcfd84eb369d011a0951d03c984f736 2024-11-27T12:25:14,617 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127acbb2fde6b994a789666840de8c72651_5dcfd84eb369d011a0951d03c984f736 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127acbb2fde6b994a789666840de8c72651_5dcfd84eb369d011a0951d03c984f736 2024-11-27T12:25:14,618 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127c03360d845b04f4ea272059430d7ce55_5dcfd84eb369d011a0951d03c984f736 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127c03360d845b04f4ea272059430d7ce55_5dcfd84eb369d011a0951d03c984f736 2024-11-27T12:25:14,620 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127f1662ebf04ff4ffe990ddda36ea12e0f_5dcfd84eb369d011a0951d03c984f736 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127f1662ebf04ff4ffe990ddda36ea12e0f_5dcfd84eb369d011a0951d03c984f736 2024-11-27T12:25:14,621 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127f534c3798cf848fda669328c66d0a30f_5dcfd84eb369d011a0951d03c984f736 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127f534c3798cf848fda669328c66d0a30f_5dcfd84eb369d011a0951d03c984f736 2024-11-27T12:25:14,621 DEBUG [PEWorker-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-27T12:25:14,623 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=58, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-27T12:25:14,626 WARN [PEWorker-2 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-27T12:25:14,629 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-27T12:25:14,630 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=58, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-27T12:25:14,630 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-27T12:25:14,630 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732710314630"}]},"ts":"9223372036854775807"} 2024-11-27T12:25:14,633 INFO [PEWorker-2 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-27T12:25:14,633 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 5dcfd84eb369d011a0951d03c984f736, NAME => 'TestAcidGuarantees,,1732710280086.5dcfd84eb369d011a0951d03c984f736.', STARTKEY => '', ENDKEY => ''}] 2024-11-27T12:25:14,633 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-27T12:25:14,633 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732710314633"}]},"ts":"9223372036854775807"} 2024-11-27T12:25:14,635 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-27T12:25:14,636 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(133): Finished pid=58, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-27T12:25:14,638 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=58, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 77 msec 2024-11-27T12:25:14,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-27T12:25:14,669 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-27T12:25:14,672 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-27T12:25:14,680 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMobMixedAtomicity Thread=238 (was 237) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/16bc5e44-ed69-d7f1-cacd-0b5be2f2db9b/cluster_2674ffaa-0f8a-9abb-ffed-bcc8bba3b369/data/data2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1180310523_22 at /127.0.0.1:37466 [Waiting for operation #15] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/16bc5e44-ed69-d7f1-cacd-0b5be2f2db9b/cluster_2674ffaa-0f8a-9abb-ffed-bcc8bba3b369/data/data1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=449 (was 445) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=416 (was 402) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=9350 (was 8876) - AvailableMemoryMB LEAK? - 2024-11-27T12:25:14,691 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testGetAtomicity Thread=238, OpenFileDescriptor=449, MaxFileDescriptor=1048576, SystemLoadAverage=416, ProcessCount=11, AvailableMemoryMB=9349 2024-11-27T12:25:14,692 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-27T12:25:14,693 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-27T12:25:14,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] procedure2.ProcedureExecutor(1139): Stored pid=59, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-27T12:25:14,695 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=59, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-27T12:25:14,695 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:25:14,696 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 59 2024-11-27T12:25:14,696 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=59, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-27T12:25:14,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=59 2024-11-27T12:25:14,702 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742055_1231 (size=959) 2024-11-27T12:25:14,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=59 2024-11-27T12:25:15,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=59 2024-11-27T12:25:15,105 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 3c3055e4b5609cf4625dfa6b13256b83, NAME => 'TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c 2024-11-27T12:25:15,111 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742056_1232 (size=53) 2024-11-27T12:25:15,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=59 2024-11-27T12:25:15,512 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T12:25:15,512 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 3c3055e4b5609cf4625dfa6b13256b83, disabling compactions & flushes 2024-11-27T12:25:15,512 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83. 2024-11-27T12:25:15,512 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83. 2024-11-27T12:25:15,512 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83. after waiting 0 ms 2024-11-27T12:25:15,512 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83. 2024-11-27T12:25:15,512 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83. 2024-11-27T12:25:15,512 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 3c3055e4b5609cf4625dfa6b13256b83: Waiting for close lock at 1732710315512Disabling compacts and flushes for region at 1732710315512Disabling writes for close at 1732710315512Writing region close event to WAL at 1732710315512Closed at 1732710315512 2024-11-27T12:25:15,513 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=59, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-27T12:25:15,514 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732710315513"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732710315513"}]},"ts":"1732710315513"} 2024-11-27T12:25:15,515 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-27T12:25:15,516 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=59, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-27T12:25:15,516 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732710315516"}]},"ts":"1732710315516"} 2024-11-27T12:25:15,517 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-27T12:25:15,518 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=60, ppid=59, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=3c3055e4b5609cf4625dfa6b13256b83, ASSIGN}] 2024-11-27T12:25:15,519 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=60, ppid=59, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=3c3055e4b5609cf4625dfa6b13256b83, ASSIGN 2024-11-27T12:25:15,519 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=60, ppid=59, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=3c3055e4b5609cf4625dfa6b13256b83, ASSIGN; state=OFFLINE, location=1ba5af43fd99,36251,1732710241631; forceNewPlan=false, retain=false 2024-11-27T12:25:15,670 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=60 updating hbase:meta row=3c3055e4b5609cf4625dfa6b13256b83, regionState=OPENING, regionLocation=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:15,676 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=60, ppid=59, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=3c3055e4b5609cf4625dfa6b13256b83, ASSIGN because future has completed 2024-11-27T12:25:15,676 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=61, ppid=60, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631}] 2024-11-27T12:25:15,831 INFO [RS_OPEN_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83. 2024-11-27T12:25:15,832 DEBUG [RS_OPEN_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(7752): Opening region: {ENCODED => 3c3055e4b5609cf4625dfa6b13256b83, NAME => 'TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83.', STARTKEY => '', ENDKEY => ''} 2024-11-27T12:25:15,832 DEBUG [RS_OPEN_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 3c3055e4b5609cf4625dfa6b13256b83 2024-11-27T12:25:15,832 DEBUG [RS_OPEN_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T12:25:15,832 DEBUG [RS_OPEN_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(7794): checking encryption for 3c3055e4b5609cf4625dfa6b13256b83 2024-11-27T12:25:15,832 DEBUG [RS_OPEN_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(7797): checking classloading for 3c3055e4b5609cf4625dfa6b13256b83 2024-11-27T12:25:15,834 INFO [StoreOpener-3c3055e4b5609cf4625dfa6b13256b83-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 3c3055e4b5609cf4625dfa6b13256b83 2024-11-27T12:25:15,835 INFO [StoreOpener-3c3055e4b5609cf4625dfa6b13256b83-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3c3055e4b5609cf4625dfa6b13256b83 columnFamilyName A 2024-11-27T12:25:15,835 DEBUG [StoreOpener-3c3055e4b5609cf4625dfa6b13256b83-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:25:15,836 INFO [StoreOpener-3c3055e4b5609cf4625dfa6b13256b83-1 {}] regionserver.HStore(327): Store=3c3055e4b5609cf4625dfa6b13256b83/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T12:25:15,836 INFO [StoreOpener-3c3055e4b5609cf4625dfa6b13256b83-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 3c3055e4b5609cf4625dfa6b13256b83 2024-11-27T12:25:15,837 INFO [StoreOpener-3c3055e4b5609cf4625dfa6b13256b83-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3c3055e4b5609cf4625dfa6b13256b83 columnFamilyName B 2024-11-27T12:25:15,837 DEBUG [StoreOpener-3c3055e4b5609cf4625dfa6b13256b83-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:25:15,838 INFO [StoreOpener-3c3055e4b5609cf4625dfa6b13256b83-1 {}] regionserver.HStore(327): Store=3c3055e4b5609cf4625dfa6b13256b83/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T12:25:15,838 INFO [StoreOpener-3c3055e4b5609cf4625dfa6b13256b83-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 3c3055e4b5609cf4625dfa6b13256b83 2024-11-27T12:25:15,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=59 2024-11-27T12:25:15,839 INFO [StoreOpener-3c3055e4b5609cf4625dfa6b13256b83-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3c3055e4b5609cf4625dfa6b13256b83 columnFamilyName C 2024-11-27T12:25:15,839 DEBUG [StoreOpener-3c3055e4b5609cf4625dfa6b13256b83-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:25:15,839 INFO [StoreOpener-3c3055e4b5609cf4625dfa6b13256b83-1 {}] regionserver.HStore(327): Store=3c3055e4b5609cf4625dfa6b13256b83/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T12:25:15,840 DEBUG [RS_OPEN_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1038): replaying wal for 3c3055e4b5609cf4625dfa6b13256b83 2024-11-27T12:25:15,840 DEBUG [RS_OPEN_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83 2024-11-27T12:25:15,841 DEBUG [RS_OPEN_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83 2024-11-27T12:25:15,842 DEBUG [RS_OPEN_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1048): stopping wal replay for 3c3055e4b5609cf4625dfa6b13256b83 2024-11-27T12:25:15,842 DEBUG [RS_OPEN_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1060): Cleaning up temporary data for 3c3055e4b5609cf4625dfa6b13256b83 2024-11-27T12:25:15,842 DEBUG [RS_OPEN_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-27T12:25:15,843 DEBUG [RS_OPEN_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1093): writing seq id for 3c3055e4b5609cf4625dfa6b13256b83 2024-11-27T12:25:15,845 DEBUG [RS_OPEN_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-27T12:25:15,845 INFO [RS_OPEN_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1114): Opened 3c3055e4b5609cf4625dfa6b13256b83; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67238573, jitterRate=0.0019328147172927856}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-27T12:25:15,846 DEBUG [RS_OPEN_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 3c3055e4b5609cf4625dfa6b13256b83 2024-11-27T12:25:15,846 DEBUG [RS_OPEN_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1006): Region open journal for 3c3055e4b5609cf4625dfa6b13256b83: Running coprocessor pre-open hook at 1732710315832Writing region info on filesystem at 1732710315832Initializing all the Stores at 1732710315833 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732710315833Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732710315833Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732710315833Cleaning up temporary data from old regions at 1732710315842 (+9 ms)Running coprocessor post-open hooks at 1732710315846 (+4 ms)Region opened successfully at 1732710315846 2024-11-27T12:25:15,847 INFO [RS_OPEN_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., pid=61, masterSystemTime=1732710315828 2024-11-27T12:25:15,849 DEBUG [RS_OPEN_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83. 2024-11-27T12:25:15,849 INFO [RS_OPEN_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83. 2024-11-27T12:25:15,849 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=60 updating hbase:meta row=3c3055e4b5609cf4625dfa6b13256b83, regionState=OPEN, openSeqNum=2, regionLocation=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:15,851 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=61, ppid=60, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 because future has completed 2024-11-27T12:25:15,853 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=61, resume processing ppid=60 2024-11-27T12:25:15,853 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 in 175 msec 2024-11-27T12:25:15,855 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=60, resume processing ppid=59 2024-11-27T12:25:15,855 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=60, ppid=59, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=3c3055e4b5609cf4625dfa6b13256b83, ASSIGN in 335 msec 2024-11-27T12:25:15,856 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=59, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-27T12:25:15,856 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732710315856"}]},"ts":"1732710315856"} 2024-11-27T12:25:15,857 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-27T12:25:15,858 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=59, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-27T12:25:15,860 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=59, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1650 sec 2024-11-27T12:25:16,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=59 2024-11-27T12:25:16,849 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-27T12:25:16,850 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1a3595f2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T12:25:16,850 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1ba5af43fd99,36215,-1 for getting cluster id 2024-11-27T12:25:16,850 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T12:25:16,851 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '200978c2-2c41-4f93-b0b5-127ea1024f37' 2024-11-27T12:25:16,851 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T12:25:16,851 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "200978c2-2c41-4f93-b0b5-127ea1024f37" 2024-11-27T12:25:16,852 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@40d21f23, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T12:25:16,852 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1ba5af43fd99,36215,-1] 2024-11-27T12:25:16,852 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T12:25:16,852 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T12:25:16,853 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39336, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T12:25:16,854 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4d34aef0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T12:25:16,854 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T12:25:16,855 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1ba5af43fd99,36251,1732710241631, seqNum=-1] 2024-11-27T12:25:16,855 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T12:25:16,856 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38892, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T12:25:16,857 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-27T12:25:16,857 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncConnectionImpl(321): The fetched master address is 1ba5af43fd99,36215,1732710240860 2024-11-27T12:25:16,857 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@4364f5a 2024-11-27T12:25:16,857 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-27T12:25:16,858 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39348, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-27T12:25:16,860 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@66263c67, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T12:25:16,860 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1ba5af43fd99,36215,-1 for getting cluster id 2024-11-27T12:25:16,860 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T12:25:16,860 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '200978c2-2c41-4f93-b0b5-127ea1024f37' 2024-11-27T12:25:16,860 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T12:25:16,861 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "200978c2-2c41-4f93-b0b5-127ea1024f37" 2024-11-27T12:25:16,861 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@317656b5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T12:25:16,861 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1ba5af43fd99,36215,-1] 2024-11-27T12:25:16,861 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T12:25:16,861 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T12:25:16,862 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39380, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T12:25:16,862 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@38ef17a9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T12:25:16,863 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4ec3b0ba, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T12:25:16,863 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1ba5af43fd99,36215,-1 for getting cluster id 2024-11-27T12:25:16,863 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T12:25:16,864 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '200978c2-2c41-4f93-b0b5-127ea1024f37' 2024-11-27T12:25:16,864 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T12:25:16,864 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "200978c2-2c41-4f93-b0b5-127ea1024f37" 2024-11-27T12:25:16,864 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7ee7c404, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T12:25:16,864 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1ba5af43fd99,36215,-1] 2024-11-27T12:25:16,864 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T12:25:16,864 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T12:25:16,865 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39392, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T12:25:16,865 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5ae2a6b6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T12:25:16,866 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@42cbc4a9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T12:25:16,866 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1ba5af43fd99,36215,-1 for getting cluster id 2024-11-27T12:25:16,866 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T12:25:16,867 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '200978c2-2c41-4f93-b0b5-127ea1024f37' 2024-11-27T12:25:16,867 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T12:25:16,867 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "200978c2-2c41-4f93-b0b5-127ea1024f37" 2024-11-27T12:25:16,867 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4838fc9f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T12:25:16,867 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1ba5af43fd99,36215,-1] 2024-11-27T12:25:16,867 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T12:25:16,867 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T12:25:16,868 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39412, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T12:25:16,868 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2e2041a1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T12:25:16,869 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4365a39d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T12:25:16,869 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1ba5af43fd99,36215,-1 for getting cluster id 2024-11-27T12:25:16,869 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T12:25:16,869 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '200978c2-2c41-4f93-b0b5-127ea1024f37' 2024-11-27T12:25:16,870 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T12:25:16,870 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "200978c2-2c41-4f93-b0b5-127ea1024f37" 2024-11-27T12:25:16,870 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2e2fc978, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T12:25:16,870 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1ba5af43fd99,36215,-1] 2024-11-27T12:25:16,870 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T12:25:16,870 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T12:25:16,871 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39420, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T12:25:16,871 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3062ca4c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T12:25:16,872 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@475d5325, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T12:25:16,872 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1ba5af43fd99,36215,-1 for getting cluster id 2024-11-27T12:25:16,872 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T12:25:16,873 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '200978c2-2c41-4f93-b0b5-127ea1024f37' 2024-11-27T12:25:16,873 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T12:25:16,874 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "200978c2-2c41-4f93-b0b5-127ea1024f37" 2024-11-27T12:25:16,874 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32a61fc9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T12:25:16,874 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1ba5af43fd99,36215,-1] 2024-11-27T12:25:16,874 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T12:25:16,874 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T12:25:16,875 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39432, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T12:25:16,876 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@fdee4b6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T12:25:16,876 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@145d04da, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T12:25:16,876 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1ba5af43fd99,36215,-1 for getting cluster id 2024-11-27T12:25:16,876 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T12:25:16,877 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '200978c2-2c41-4f93-b0b5-127ea1024f37' 2024-11-27T12:25:16,877 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T12:25:16,877 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "200978c2-2c41-4f93-b0b5-127ea1024f37" 2024-11-27T12:25:16,877 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@67a358b4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T12:25:16,878 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1ba5af43fd99,36215,-1] 2024-11-27T12:25:16,878 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T12:25:16,878 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T12:25:16,879 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39452, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T12:25:16,879 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2aef0f80, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T12:25:16,880 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@57aa347, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T12:25:16,880 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1ba5af43fd99,36215,-1 for getting cluster id 2024-11-27T12:25:16,880 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T12:25:16,881 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '200978c2-2c41-4f93-b0b5-127ea1024f37' 2024-11-27T12:25:16,881 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T12:25:16,881 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "200978c2-2c41-4f93-b0b5-127ea1024f37" 2024-11-27T12:25:16,881 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@58e24ff1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T12:25:16,881 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1ba5af43fd99,36215,-1] 2024-11-27T12:25:16,881 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T12:25:16,881 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T12:25:16,882 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39472, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T12:25:16,882 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@43d48253, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T12:25:16,883 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a3ed5e3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T12:25:16,883 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1ba5af43fd99,36215,-1 for getting cluster id 2024-11-27T12:25:16,883 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T12:25:16,884 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '200978c2-2c41-4f93-b0b5-127ea1024f37' 2024-11-27T12:25:16,884 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T12:25:16,884 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "200978c2-2c41-4f93-b0b5-127ea1024f37" 2024-11-27T12:25:16,884 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3bdc4f95, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T12:25:16,884 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1ba5af43fd99,36215,-1] 2024-11-27T12:25:16,884 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T12:25:16,884 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T12:25:16,885 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39480, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T12:25:16,886 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@330547f6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T12:25:16,886 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@66fab171, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T12:25:16,886 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1ba5af43fd99,36215,-1 for getting cluster id 2024-11-27T12:25:16,887 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T12:25:16,887 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '200978c2-2c41-4f93-b0b5-127ea1024f37' 2024-11-27T12:25:16,888 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T12:25:16,888 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "200978c2-2c41-4f93-b0b5-127ea1024f37" 2024-11-27T12:25:16,888 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@22d7afda, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T12:25:16,888 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1ba5af43fd99,36215,-1] 2024-11-27T12:25:16,888 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T12:25:16,888 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T12:25:16,889 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39500, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T12:25:16,890 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@52a9ba0a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T12:25:16,890 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@51ba3a53, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T12:25:16,890 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1ba5af43fd99,36215,-1 for getting cluster id 2024-11-27T12:25:16,890 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T12:25:16,891 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '200978c2-2c41-4f93-b0b5-127ea1024f37' 2024-11-27T12:25:16,891 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T12:25:16,891 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "200978c2-2c41-4f93-b0b5-127ea1024f37" 2024-11-27T12:25:16,891 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d87d72e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T12:25:16,891 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1ba5af43fd99,36215,-1] 2024-11-27T12:25:16,891 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T12:25:16,892 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T12:25:16,892 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39516, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T12:25:16,893 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4a2ecb11, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T12:25:16,893 DEBUG [Thread-1107 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T12:25:16,893 DEBUG [Thread-1103 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T12:25:16,893 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-27T12:25:16,893 DEBUG [Thread-1096 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T12:25:16,894 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1ba5af43fd99,36251,1732710241631, seqNum=-1] 2024-11-27T12:25:16,894 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1ba5af43fd99,36251,1732710241631, seqNum=-1] 2024-11-27T12:25:16,894 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1ba5af43fd99,36251,1732710241631, seqNum=-1] 2024-11-27T12:25:16,894 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T12:25:16,894 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T12:25:16,894 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T12:25:16,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] procedure2.ProcedureExecutor(1139): Stored pid=62, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees 2024-11-27T12:25:16,895 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38898, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T12:25:16,895 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38902, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T12:25:16,895 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38900, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T12:25:16,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-27T12:25:16,897 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=62, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T12:25:16,897 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=62, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T12:25:16,898 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=63, ppid=62, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T12:25:16,899 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2] 2024-11-27T12:25:16,900 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2] 2024-11-27T12:25:16,900 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2] 2024-11-27T12:25:16,900 DEBUG [Thread-1101 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T12:25:16,900 DEBUG [Thread-1088 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T12:25:16,901 DEBUG [Thread-1092 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T12:25:16,901 DEBUG [Thread-1105 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T12:25:16,901 DEBUG [Thread-1090 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T12:25:16,901 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1ba5af43fd99,36251,1732710241631, seqNum=-1] 2024-11-27T12:25:16,901 DEBUG [Thread-1101 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1ba5af43fd99,36251,1732710241631, seqNum=-1] 2024-11-27T12:25:16,902 DEBUG [Thread-1101 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T12:25:16,902 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T12:25:16,902 DEBUG [Thread-1092 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1ba5af43fd99,36251,1732710241631, seqNum=-1] 2024-11-27T12:25:16,902 DEBUG [Thread-1092 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T12:25:16,902 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1ba5af43fd99,36251,1732710241631, seqNum=-1] 2024-11-27T12:25:16,902 DEBUG [Thread-1105 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1ba5af43fd99,36251,1732710241631, seqNum=-1] 2024-11-27T12:25:16,902 DEBUG [Thread-1105 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T12:25:16,902 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T12:25:16,903 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38952, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T12:25:16,904 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38944, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T12:25:16,904 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38978, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T12:25:16,904 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38936, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T12:25:16,904 DEBUG [Thread-1094 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T12:25:16,904 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38966, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T12:25:16,905 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1ba5af43fd99,36251,1732710241631, seqNum=-1] 2024-11-27T12:25:16,905 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T12:25:16,906 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2] 2024-11-27T12:25:16,906 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2] 2024-11-27T12:25:16,907 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2] 2024-11-27T12:25:16,907 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2] 2024-11-27T12:25:16,907 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38986, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T12:25:16,908 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2] 2024-11-27T12:25:16,909 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2] 2024-11-27T12:25:16,912 DEBUG [Thread-1099 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T12:25:16,913 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1ba5af43fd99,36251,1732710241631, seqNum=-1] 2024-11-27T12:25:16,913 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T12:25:16,914 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39002, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T12:25:16,916 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2] 2024-11-27T12:25:16,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(8855): Flush requested on 3c3055e4b5609cf4625dfa6b13256b83 2024-11-27T12:25:16,944 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T12:25:16,944 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3c3055e4b5609cf4625dfa6b13256b83 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-27T12:25:16,971 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/.tmp/A/90f42d0014dc4971b27ced34e8b315c2 is 50, key is test_row_0/A:col10/1732710316910/Put/seqid=0 2024-11-27T12:25:16,979 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742057_1233 (size=12001) 2024-11-27T12:25:16,981 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/.tmp/A/90f42d0014dc4971b27ced34e8b315c2 2024-11-27T12:25:17,001 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:17,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38902 deadline: 1732710376990, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:17,001 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:17,002 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:17,002 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:17,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38986 deadline: 1732710376992, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:17,002 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:17,002 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:17,002 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:17,002 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:17,006 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:17,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38944 deadline: 1732710376998, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:17,007 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:17,007 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:17,007 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:17,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-27T12:25:17,009 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:17,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38978 deadline: 1732710376999, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:17,010 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:17,010 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:17,010 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:17,016 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:17,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38952 deadline: 1732710377010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:17,018 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:17,018 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:17,018 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:17,039 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/.tmp/B/afee21ebf75749a19fbc487182a483ea is 50, key is test_row_0/B:col10/1732710316910/Put/seqid=0 2024-11-27T12:25:17,050 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-27T12:25:17,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83. 2024-11-27T12:25:17,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83. as already flushing 2024-11-27T12:25:17,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83. 2024-11-27T12:25:17,051 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 3c3055e4b5609cf4625dfa6b13256b83, NAME => 'TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:25:17,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 3c3055e4b5609cf4625dfa6b13256b83, NAME => 'TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:25:17,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36215 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3c3055e4b5609cf4625dfa6b13256b83, NAME => 'TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3c3055e4b5609cf4625dfa6b13256b83, NAME => 'TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:25:17,082 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742058_1234 (size=12001) 2024-11-27T12:25:17,083 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/.tmp/B/afee21ebf75749a19fbc487182a483ea 2024-11-27T12:25:17,110 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:17,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38902 deadline: 1732710377109, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:17,110 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:17,110 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:17,110 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:17,112 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:17,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38986 deadline: 1732710377109, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:17,112 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:17,112 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:17,112 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:17,123 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:17,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38944 deadline: 1732710377120, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:17,123 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:17,123 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:17,123 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:17,124 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:17,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38978 deadline: 1732710377120, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:17,124 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:17,124 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:17,124 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:17,125 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/.tmp/C/8491cd57ae9148668e638d1315148c25 is 50, key is test_row_0/C:col10/1732710316910/Put/seqid=0 2024-11-27T12:25:17,131 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:17,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38952 deadline: 1732710377128, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:17,132 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:17,132 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:17,132 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:17,138 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742059_1235 (size=12001) 2024-11-27T12:25:17,139 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/.tmp/C/8491cd57ae9148668e638d1315148c25 2024-11-27T12:25:17,149 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/.tmp/A/90f42d0014dc4971b27ced34e8b315c2 as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/A/90f42d0014dc4971b27ced34e8b315c2 2024-11-27T12:25:17,165 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/A/90f42d0014dc4971b27ced34e8b315c2, entries=150, sequenceid=12, filesize=11.7 K 2024-11-27T12:25:17,167 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/.tmp/B/afee21ebf75749a19fbc487182a483ea as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/B/afee21ebf75749a19fbc487182a483ea 2024-11-27T12:25:17,173 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/B/afee21ebf75749a19fbc487182a483ea, entries=150, sequenceid=12, filesize=11.7 K 2024-11-27T12:25:17,175 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/.tmp/C/8491cd57ae9148668e638d1315148c25 as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/C/8491cd57ae9148668e638d1315148c25 2024-11-27T12:25:17,192 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/C/8491cd57ae9148668e638d1315148c25, entries=150, sequenceid=12, filesize=11.7 K 2024-11-27T12:25:17,193 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=154.31 KB/158010 for 3c3055e4b5609cf4625dfa6b13256b83 in 249ms, sequenceid=12, compaction requested=false 2024-11-27T12:25:17,193 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3c3055e4b5609cf4625dfa6b13256b83: 2024-11-27T12:25:17,205 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-27T12:25:17,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83. 2024-11-27T12:25:17,205 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2902): Flushing 3c3055e4b5609cf4625dfa6b13256b83 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-27T12:25:17,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/.tmp/A/f4b453f643af48b5b9bb6e24dfb518f5 is 50, key is test_row_0/A:col10/1732710316998/Put/seqid=0 2024-11-27T12:25:17,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-27T12:25:17,252 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742060_1236 (size=12001) 2024-11-27T12:25:17,256 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=38 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/.tmp/A/f4b453f643af48b5b9bb6e24dfb518f5 2024-11-27T12:25:17,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/.tmp/B/d83f32d2f923428d85814b904e3f3638 is 50, key is test_row_0/B:col10/1732710316998/Put/seqid=0 2024-11-27T12:25:17,306 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742061_1237 (size=12001) 2024-11-27T12:25:17,307 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=38 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/.tmp/B/d83f32d2f923428d85814b904e3f3638 2024-11-27T12:25:17,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(8855): Flush requested on 3c3055e4b5609cf4625dfa6b13256b83 2024-11-27T12:25:17,322 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83. as already flushing 2024-11-27T12:25:17,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/.tmp/C/f68317e826e649659bd372d862b3e71b is 50, key is test_row_0/C:col10/1732710316998/Put/seqid=0 2024-11-27T12:25:17,341 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:17,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38986 deadline: 1732710377334, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:17,341 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:17,341 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:17,341 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:17,342 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:17,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38978 deadline: 1732710377335, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:17,342 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:17,342 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:17,342 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:17,342 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:17,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38944 deadline: 1732710377336, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:17,343 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:17,343 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:17,343 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:17,343 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:17,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38902 deadline: 1732710377336, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:17,343 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:17,343 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:17,343 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:17,343 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:17,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38952 deadline: 1732710377338, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:17,344 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:17,344 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:17,344 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:17,346 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742062_1238 (size=12001) 2024-11-27T12:25:17,349 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=38 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/.tmp/C/f68317e826e649659bd372d862b3e71b 2024-11-27T12:25:17,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/.tmp/A/f4b453f643af48b5b9bb6e24dfb518f5 as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/A/f4b453f643af48b5b9bb6e24dfb518f5 2024-11-27T12:25:17,363 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/A/f4b453f643af48b5b9bb6e24dfb518f5, entries=150, sequenceid=38, filesize=11.7 K 2024-11-27T12:25:17,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/.tmp/B/d83f32d2f923428d85814b904e3f3638 as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/B/d83f32d2f923428d85814b904e3f3638 2024-11-27T12:25:17,372 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/B/d83f32d2f923428d85814b904e3f3638, entries=150, sequenceid=38, filesize=11.7 K 2024-11-27T12:25:17,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/.tmp/C/f68317e826e649659bd372d862b3e71b as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/C/f68317e826e649659bd372d862b3e71b 2024-11-27T12:25:17,382 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/C/f68317e826e649659bd372d862b3e71b, entries=150, sequenceid=38, filesize=11.7 K 2024-11-27T12:25:17,384 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=53.67 KB/54960 for 3c3055e4b5609cf4625dfa6b13256b83 in 179ms, sequenceid=38, compaction requested=false 2024-11-27T12:25:17,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2603): Flush status journal for 3c3055e4b5609cf4625dfa6b13256b83: 2024-11-27T12:25:17,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83. 2024-11-27T12:25:17,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=63 2024-11-27T12:25:17,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36215 {}] master.HMaster(4169): Remote procedure done, pid=63 2024-11-27T12:25:17,390 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=63, resume processing ppid=62 2024-11-27T12:25:17,390 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=63, ppid=62, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 488 msec 2024-11-27T12:25:17,394 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=62, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees in 498 msec 2024-11-27T12:25:17,453 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T12:25:17,453 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3c3055e4b5609cf4625dfa6b13256b83 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-27T12:25:17,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(8855): Flush requested on 3c3055e4b5609cf4625dfa6b13256b83 2024-11-27T12:25:17,474 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/.tmp/A/fc30337f9fa3475db6fbc578666ebc96 is 50, key is test_row_0/A:col10/1732710317334/Put/seqid=0 2024-11-27T12:25:17,490 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742063_1239 (size=14341) 2024-11-27T12:25:17,490 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/.tmp/A/fc30337f9fa3475db6fbc578666ebc96 2024-11-27T12:25:17,516 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:17,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38986 deadline: 1732710377509, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:17,516 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:17,517 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:17,517 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:17,517 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:17,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38902 deadline: 1732710377513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:17,518 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:17,519 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:17,519 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:17,520 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/.tmp/B/ce147e8dccaa433d9649ec8cd42a7811 is 50, key is test_row_0/B:col10/1732710317334/Put/seqid=0 2024-11-27T12:25:17,521 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:17,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38944 deadline: 1732710377516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:17,522 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:17,522 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:17,522 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:17,524 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:17,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38978 deadline: 1732710377513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:17,525 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:17,525 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:17,525 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:17,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-27T12:25:17,529 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T12:25:17,529 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36215 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-27T12:25:17,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36215 {}] procedure2.ProcedureExecutor(1139): Stored pid=64, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees 2024-11-27T12:25:17,531 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=64, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T12:25:17,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-27T12:25:17,532 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=64, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T12:25:17,532 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=65, ppid=64, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T12:25:17,567 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742064_1240 (size=12001) 2024-11-27T12:25:17,631 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:17,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38986 deadline: 1732710377628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:17,632 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:17,632 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:17,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38978 deadline: 1732710377629, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:17,632 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:17,632 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:17,632 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:17,632 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:17,632 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:17,632 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:17,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38902 deadline: 1732710377629, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:17,633 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:17,633 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:17,633 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:17,633 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:17,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38944 deadline: 1732710377629, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:17,633 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:17,633 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:17,633 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:17,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-27T12:25:17,650 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:17,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38952 deadline: 1732710377648, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:17,651 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:17,651 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:17,651 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:17,685 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-27T12:25:17,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83. 2024-11-27T12:25:17,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83. as already flushing 2024-11-27T12:25:17,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83. 2024-11-27T12:25:17,685 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 3c3055e4b5609cf4625dfa6b13256b83, NAME => 'TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:25:17,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 3c3055e4b5609cf4625dfa6b13256b83, NAME => 'TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:25:17,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36215 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3c3055e4b5609cf4625dfa6b13256b83, NAME => 'TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3c3055e4b5609cf4625dfa6b13256b83, NAME => 'TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:25:17,839 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:17,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38986 deadline: 1732710377838, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:17,840 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:17,840 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:17,840 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:17,840 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:17,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38978 deadline: 1732710377838, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:17,840 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-27T12:25:17,840 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:17,840 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:17,840 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:17,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83. 2024-11-27T12:25:17,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83. as already flushing 2024-11-27T12:25:17,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83. 2024-11-27T12:25:17,841 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 3c3055e4b5609cf4625dfa6b13256b83, NAME => 'TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:25:17,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 3c3055e4b5609cf4625dfa6b13256b83, NAME => 'TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:25:17,841 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:17,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38944 deadline: 1732710377838, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:17,841 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:17,841 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:17,841 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:17,841 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:17,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38902 deadline: 1732710377838, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:17,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36215 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3c3055e4b5609cf4625dfa6b13256b83, NAME => 'TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3c3055e4b5609cf4625dfa6b13256b83, NAME => 'TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:25:17,841 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:17,841 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:17,841 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:17,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-27T12:25:17,967 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/.tmp/B/ce147e8dccaa433d9649ec8cd42a7811 2024-11-27T12:25:17,984 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/.tmp/C/3b471d6df3f84234b24672a12e747fb6 is 50, key is test_row_0/C:col10/1732710317334/Put/seqid=0 2024-11-27T12:25:17,993 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-27T12:25:17,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83. 2024-11-27T12:25:17,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83. as already flushing 2024-11-27T12:25:17,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83. 2024-11-27T12:25:17,993 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 3c3055e4b5609cf4625dfa6b13256b83, NAME => 'TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:25:17,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 3c3055e4b5609cf4625dfa6b13256b83, NAME => 'TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:25:17,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36215 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3c3055e4b5609cf4625dfa6b13256b83, NAME => 'TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3c3055e4b5609cf4625dfa6b13256b83, NAME => 'TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:25:18,004 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742065_1241 (size=12001) 2024-11-27T12:25:18,010 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/.tmp/C/3b471d6df3f84234b24672a12e747fb6 2024-11-27T12:25:18,015 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/.tmp/A/fc30337f9fa3475db6fbc578666ebc96 as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/A/fc30337f9fa3475db6fbc578666ebc96 2024-11-27T12:25:18,020 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/A/fc30337f9fa3475db6fbc578666ebc96, entries=200, sequenceid=51, filesize=14.0 K 2024-11-27T12:25:18,021 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/.tmp/B/ce147e8dccaa433d9649ec8cd42a7811 as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/B/ce147e8dccaa433d9649ec8cd42a7811 2024-11-27T12:25:18,026 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/B/ce147e8dccaa433d9649ec8cd42a7811, entries=150, sequenceid=51, filesize=11.7 K 2024-11-27T12:25:18,027 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/.tmp/C/3b471d6df3f84234b24672a12e747fb6 as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/C/3b471d6df3f84234b24672a12e747fb6 2024-11-27T12:25:18,030 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/C/3b471d6df3f84234b24672a12e747fb6, entries=150, sequenceid=51, filesize=11.7 K 2024-11-27T12:25:18,031 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 3c3055e4b5609cf4625dfa6b13256b83 in 578ms, sequenceid=51, compaction requested=true 2024-11-27T12:25:18,031 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3c3055e4b5609cf4625dfa6b13256b83: 2024-11-27T12:25:18,031 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3c3055e4b5609cf4625dfa6b13256b83:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T12:25:18,031 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T12:25:18,031 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3c3055e4b5609cf4625dfa6b13256b83:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T12:25:18,031 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T12:25:18,031 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T12:25:18,031 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3c3055e4b5609cf4625dfa6b13256b83:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T12:25:18,031 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T12:25:18,031 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T12:25:18,032 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38343 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T12:25:18,032 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HStore(1541): 3c3055e4b5609cf4625dfa6b13256b83/A is initiating minor compaction (all files) 2024-11-27T12:25:18,033 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3c3055e4b5609cf4625dfa6b13256b83/A in TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83. 2024-11-27T12:25:18,033 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/A/90f42d0014dc4971b27ced34e8b315c2, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/A/f4b453f643af48b5b9bb6e24dfb518f5, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/A/fc30337f9fa3475db6fbc578666ebc96] into tmpdir=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/.tmp, totalSize=37.4 K 2024-11-27T12:25:18,033 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T12:25:18,033 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HStore(1541): 3c3055e4b5609cf4625dfa6b13256b83/B is initiating minor compaction (all files) 2024-11-27T12:25:18,033 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] compactions.Compactor(225): Compacting 90f42d0014dc4971b27ced34e8b315c2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1732710316907 2024-11-27T12:25:18,033 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3c3055e4b5609cf4625dfa6b13256b83/B in TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83. 2024-11-27T12:25:18,033 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/B/afee21ebf75749a19fbc487182a483ea, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/B/d83f32d2f923428d85814b904e3f3638, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/B/ce147e8dccaa433d9649ec8cd42a7811] into tmpdir=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/.tmp, totalSize=35.2 K 2024-11-27T12:25:18,033 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.Compactor(225): Compacting afee21ebf75749a19fbc487182a483ea, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1732710316907 2024-11-27T12:25:18,034 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] compactions.Compactor(225): Compacting f4b453f643af48b5b9bb6e24dfb518f5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=38, earliestPutTs=1732710316991 2024-11-27T12:25:18,034 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.Compactor(225): Compacting d83f32d2f923428d85814b904e3f3638, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=38, earliestPutTs=1732710316991 2024-11-27T12:25:18,034 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] compactions.Compactor(225): Compacting fc30337f9fa3475db6fbc578666ebc96, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1732710317321 2024-11-27T12:25:18,034 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.Compactor(225): Compacting ce147e8dccaa433d9649ec8cd42a7811, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1732710317321 2024-11-27T12:25:18,051 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3c3055e4b5609cf4625dfa6b13256b83#B#compaction#196 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-27T12:25:18,051 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/.tmp/B/cfdf1edf206c48839298e7083fdec231 is 50, key is test_row_0/B:col10/1732710317334/Put/seqid=0 2024-11-27T12:25:18,051 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3c3055e4b5609cf4625dfa6b13256b83#A#compaction#195 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T12:25:18,052 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/.tmp/A/67e8e65a784045389118c6db13ad5655 is 50, key is test_row_0/A:col10/1732710317334/Put/seqid=0 2024-11-27T12:25:18,065 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742066_1242 (size=12104) 2024-11-27T12:25:18,072 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/.tmp/B/cfdf1edf206c48839298e7083fdec231 as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/B/cfdf1edf206c48839298e7083fdec231 2024-11-27T12:25:18,107 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3c3055e4b5609cf4625dfa6b13256b83/B of 3c3055e4b5609cf4625dfa6b13256b83 into cfdf1edf206c48839298e7083fdec231(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T12:25:18,107 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3c3055e4b5609cf4625dfa6b13256b83: 2024-11-27T12:25:18,107 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., storeName=3c3055e4b5609cf4625dfa6b13256b83/B, priority=13, startTime=1732710318031; duration=0sec 2024-11-27T12:25:18,107 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T12:25:18,107 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3c3055e4b5609cf4625dfa6b13256b83:B 2024-11-27T12:25:18,108 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T12:25:18,110 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742067_1243 (size=12104) 2024-11-27T12:25:18,110 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T12:25:18,110 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HStore(1541): 3c3055e4b5609cf4625dfa6b13256b83/C is initiating minor compaction (all files) 2024-11-27T12:25:18,110 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3c3055e4b5609cf4625dfa6b13256b83/C in TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83. 2024-11-27T12:25:18,111 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/C/8491cd57ae9148668e638d1315148c25, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/C/f68317e826e649659bd372d862b3e71b, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/C/3b471d6df3f84234b24672a12e747fb6] into tmpdir=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/.tmp, totalSize=35.2 K 2024-11-27T12:25:18,111 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.Compactor(225): Compacting 8491cd57ae9148668e638d1315148c25, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1732710316907 2024-11-27T12:25:18,112 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.Compactor(225): Compacting f68317e826e649659bd372d862b3e71b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=38, earliestPutTs=1732710316991 2024-11-27T12:25:18,112 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.Compactor(225): Compacting 3b471d6df3f84234b24672a12e747fb6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1732710317321 2024-11-27T12:25:18,127 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/.tmp/A/67e8e65a784045389118c6db13ad5655 as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/A/67e8e65a784045389118c6db13ad5655 2024-11-27T12:25:18,138 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3c3055e4b5609cf4625dfa6b13256b83/A of 3c3055e4b5609cf4625dfa6b13256b83 into 67e8e65a784045389118c6db13ad5655(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T12:25:18,138 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3c3055e4b5609cf4625dfa6b13256b83: 2024-11-27T12:25:18,138 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., storeName=3c3055e4b5609cf4625dfa6b13256b83/A, priority=13, startTime=1732710318031; duration=0sec 2024-11-27T12:25:18,140 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T12:25:18,140 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3c3055e4b5609cf4625dfa6b13256b83:A 2024-11-27T12:25:18,146 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-27T12:25:18,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83. 2024-11-27T12:25:18,146 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2902): Flushing 3c3055e4b5609cf4625dfa6b13256b83 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-27T12:25:18,150 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83. as already flushing 2024-11-27T12:25:18,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(8855): Flush requested on 3c3055e4b5609cf4625dfa6b13256b83 2024-11-27T12:25:18,151 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3c3055e4b5609cf4625dfa6b13256b83#C#compaction#197 average throughput is 1.09 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T12:25:18,151 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/.tmp/C/06c9741688a7405884a3aa541a3226da is 50, key is test_row_0/C:col10/1732710317334/Put/seqid=0 2024-11-27T12:25:18,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-27T12:25:18,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/.tmp/A/010d9b792c054cd8b0aefc3ab09c2fbf is 50, key is test_row_0/A:col10/1732710317505/Put/seqid=0 2024-11-27T12:25:18,169 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:18,169 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:18,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38944 deadline: 1732710378159, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:18,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38902 deadline: 1732710378160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:18,169 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:18,169 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:18,169 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:18,170 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:18,170 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:18,170 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:18,174 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:18,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38986 deadline: 1732710378169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:18,175 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:18,175 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:18,175 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:18,175 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:18,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38978 deadline: 1732710378169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:18,176 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:18,176 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:18,176 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:18,176 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:18,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38952 deadline: 1732710378170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:18,176 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:18,177 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:18,177 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:18,214 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742068_1244 (size=12104) 2024-11-27T12:25:18,236 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742069_1245 (size=12001) 2024-11-27T12:25:18,281 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:18,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38944 deadline: 1732710378278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:18,281 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:18,281 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:18,281 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:18,281 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:18,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38986 deadline: 1732710378279, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:18,282 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:18,282 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:18,282 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:18,282 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:18,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38902 deadline: 1732710378279, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:18,282 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:18,282 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:18,282 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:18,283 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:18,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38978 deadline: 1732710378279, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:18,283 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:18,283 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:18,283 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:18,294 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:18,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38952 deadline: 1732710378289, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:18,294 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:18,295 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:18,295 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:18,490 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:18,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38944 deadline: 1732710378488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:18,491 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:18,491 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:18,491 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:18,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38986 deadline: 1732710378488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:18,491 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:18,491 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:18,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38902 deadline: 1732710378488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:18,491 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:18,491 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:18,491 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:18,491 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:18,491 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:18,491 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:18,491 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:18,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38978 deadline: 1732710378488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:18,491 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:18,492 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:18,492 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:18,500 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:18,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38952 deadline: 1732710378498, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:18,500 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:18,500 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:18,500 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:18,622 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/.tmp/C/06c9741688a7405884a3aa541a3226da as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/C/06c9741688a7405884a3aa541a3226da 2024-11-27T12:25:18,627 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3c3055e4b5609cf4625dfa6b13256b83/C of 3c3055e4b5609cf4625dfa6b13256b83 into 06c9741688a7405884a3aa541a3226da(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T12:25:18,627 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3c3055e4b5609cf4625dfa6b13256b83: 2024-11-27T12:25:18,627 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., storeName=3c3055e4b5609cf4625dfa6b13256b83/C, priority=13, startTime=1732710318031; duration=0sec 2024-11-27T12:25:18,627 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T12:25:18,627 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3c3055e4b5609cf4625dfa6b13256b83:C 2024-11-27T12:25:18,637 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=76 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/.tmp/A/010d9b792c054cd8b0aefc3ab09c2fbf 2024-11-27T12:25:18,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/.tmp/B/842a0a725343457b9486270b91a3f6c2 is 50, key is test_row_0/B:col10/1732710317505/Put/seqid=0 2024-11-27T12:25:18,647 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742070_1246 (size=12001) 2024-11-27T12:25:18,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-27T12:25:18,800 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:18,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38944 deadline: 1732710378798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:18,801 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:18,801 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:18,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38986 deadline: 1732710378798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:18,801 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:18,801 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:18,801 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:18,801 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:18,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38902 deadline: 1732710378798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:18,801 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:18,801 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:18,801 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:18,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38978 deadline: 1732710378798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:18,802 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:18,802 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:18,802 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:18,802 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:18,802 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:18,802 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:18,811 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:18,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38952 deadline: 1732710378808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:18,811 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:18,811 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:18,811 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:19,048 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=76 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/.tmp/B/842a0a725343457b9486270b91a3f6c2 2024-11-27T12:25:19,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/.tmp/C/6081cd35ef12471787e429634907855f is 50, key is test_row_0/C:col10/1732710317505/Put/seqid=0 2024-11-27T12:25:19,064 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742071_1247 (size=12001) 2024-11-27T12:25:19,065 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=76 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/.tmp/C/6081cd35ef12471787e429634907855f 2024-11-27T12:25:19,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/.tmp/A/010d9b792c054cd8b0aefc3ab09c2fbf as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/A/010d9b792c054cd8b0aefc3ab09c2fbf 2024-11-27T12:25:19,073 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/A/010d9b792c054cd8b0aefc3ab09c2fbf, entries=150, sequenceid=76, filesize=11.7 K 2024-11-27T12:25:19,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/.tmp/B/842a0a725343457b9486270b91a3f6c2 as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/B/842a0a725343457b9486270b91a3f6c2 2024-11-27T12:25:19,078 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/B/842a0a725343457b9486270b91a3f6c2, entries=150, sequenceid=76, filesize=11.7 K 2024-11-27T12:25:19,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/.tmp/C/6081cd35ef12471787e429634907855f as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/C/6081cd35ef12471787e429634907855f 2024-11-27T12:25:19,084 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/C/6081cd35ef12471787e429634907855f, entries=150, sequenceid=76, filesize=11.7 K 2024-11-27T12:25:19,085 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=73.80 KB/75570 for 3c3055e4b5609cf4625dfa6b13256b83 in 939ms, sequenceid=76, compaction requested=false 2024-11-27T12:25:19,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2603): Flush status journal for 3c3055e4b5609cf4625dfa6b13256b83: 2024-11-27T12:25:19,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83. 2024-11-27T12:25:19,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=65 2024-11-27T12:25:19,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36215 {}] master.HMaster(4169): Remote procedure done, pid=65 2024-11-27T12:25:19,089 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=65, resume processing ppid=64 2024-11-27T12:25:19,089 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=65, ppid=64, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5550 sec 2024-11-27T12:25:19,091 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=64, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees in 1.5610 sec 2024-11-27T12:25:19,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(8855): Flush requested on 3c3055e4b5609cf4625dfa6b13256b83 2024-11-27T12:25:19,310 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T12:25:19,310 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3c3055e4b5609cf4625dfa6b13256b83 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-27T12:25:19,316 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/.tmp/A/bcc01fb81baa40eda70649ba9b89ad82 is 50, key is test_row_0/A:col10/1732710319309/Put/seqid=0 2024-11-27T12:25:19,320 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742072_1248 (size=12001) 2024-11-27T12:25:19,320 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=93 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/.tmp/A/bcc01fb81baa40eda70649ba9b89ad82 2024-11-27T12:25:19,332 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:19,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38952 deadline: 1732710379328, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:19,333 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:19,333 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:19,333 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:19,334 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:19,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38978 deadline: 1732710379330, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:19,335 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:19,335 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:19,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38986 deadline: 1732710379330, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:19,335 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:19,335 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:19,335 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:19,335 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:19,335 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:19,335 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:19,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38902 deadline: 1732710379331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:19,336 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:19,336 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:19,336 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:19,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38944 deadline: 1732710379332, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:19,336 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:19,336 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:19,336 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:19,336 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:19,342 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/.tmp/B/1126a40417a341cbb1db114f53cbb579 is 50, key is test_row_0/B:col10/1732710319309/Put/seqid=0 2024-11-27T12:25:19,361 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742073_1249 (size=12001) 2024-11-27T12:25:19,362 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=93 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/.tmp/B/1126a40417a341cbb1db114f53cbb579 2024-11-27T12:25:19,371 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/.tmp/C/ed0015bbefa8428fb4ccd28e6859ebe7 is 50, key is test_row_0/C:col10/1732710319309/Put/seqid=0 2024-11-27T12:25:19,388 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742074_1250 (size=12001) 2024-11-27T12:25:19,442 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:19,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38952 deadline: 1732710379438, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:19,442 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:19,442 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:19,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38902 deadline: 1732710379439, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:19,442 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:19,442 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:19,443 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:19,443 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:19,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38986 deadline: 1732710379439, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:19,443 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:19,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38978 deadline: 1732710379439, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:19,443 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:19,443 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:19,443 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:19,443 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:19,443 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:19,443 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:19,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38944 deadline: 1732710379439, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:19,443 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:19,443 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:19,443 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:19,443 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:19,443 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:19,443 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:19,541 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-27T12:25:19,651 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:19,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38952 deadline: 1732710379648, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:19,652 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:19,652 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:19,652 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:19,652 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:19,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38986 deadline: 1732710379648, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:19,652 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:19,652 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:19,652 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:19,652 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:19,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38902 deadline: 1732710379648, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:19,652 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:19,653 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:19,653 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:19,653 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:19,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38944 deadline: 1732710379648, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:19,653 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:19,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38978 deadline: 1732710379648, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:19,653 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:19,653 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:19,653 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:19,654 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:19,654 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:19,654 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:19,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-27T12:25:19,688 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T12:25:19,688 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36215 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-27T12:25:19,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36215 {}] procedure2.ProcedureExecutor(1139): Stored pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees 2024-11-27T12:25:19,690 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T12:25:19,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-27T12:25:19,691 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T12:25:19,691 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=67, ppid=66, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T12:25:19,789 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=93 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/.tmp/C/ed0015bbefa8428fb4ccd28e6859ebe7 2024-11-27T12:25:19,794 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/.tmp/A/bcc01fb81baa40eda70649ba9b89ad82 as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/A/bcc01fb81baa40eda70649ba9b89ad82 2024-11-27T12:25:19,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-27T12:25:19,799 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/A/bcc01fb81baa40eda70649ba9b89ad82, entries=150, sequenceid=93, filesize=11.7 K 2024-11-27T12:25:19,804 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/.tmp/B/1126a40417a341cbb1db114f53cbb579 as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/B/1126a40417a341cbb1db114f53cbb579 2024-11-27T12:25:19,809 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/B/1126a40417a341cbb1db114f53cbb579, entries=150, sequenceid=93, filesize=11.7 K 2024-11-27T12:25:19,809 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/.tmp/C/ed0015bbefa8428fb4ccd28e6859ebe7 as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/C/ed0015bbefa8428fb4ccd28e6859ebe7 2024-11-27T12:25:19,813 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/C/ed0015bbefa8428fb4ccd28e6859ebe7, entries=150, sequenceid=93, filesize=11.7 K 2024-11-27T12:25:19,814 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=114.05 KB/116790 for 3c3055e4b5609cf4625dfa6b13256b83 in 504ms, sequenceid=93, compaction requested=true 2024-11-27T12:25:19,814 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3c3055e4b5609cf4625dfa6b13256b83: 2024-11-27T12:25:19,814 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3c3055e4b5609cf4625dfa6b13256b83:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T12:25:19,814 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T12:25:19,814 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3c3055e4b5609cf4625dfa6b13256b83:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T12:25:19,814 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T12:25:19,814 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T12:25:19,814 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T12:25:19,814 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3c3055e4b5609cf4625dfa6b13256b83:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T12:25:19,815 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T12:25:19,815 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T12:25:19,815 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HStore(1541): 3c3055e4b5609cf4625dfa6b13256b83/B is initiating minor compaction (all files) 2024-11-27T12:25:19,815 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3c3055e4b5609cf4625dfa6b13256b83/B in TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83. 2024-11-27T12:25:19,816 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/B/cfdf1edf206c48839298e7083fdec231, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/B/842a0a725343457b9486270b91a3f6c2, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/B/1126a40417a341cbb1db114f53cbb579] into tmpdir=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/.tmp, totalSize=35.3 K 2024-11-27T12:25:19,816 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T12:25:19,816 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HStore(1541): 3c3055e4b5609cf4625dfa6b13256b83/A is initiating minor compaction (all files) 2024-11-27T12:25:19,816 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3c3055e4b5609cf4625dfa6b13256b83/A in TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83. 2024-11-27T12:25:19,816 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/A/67e8e65a784045389118c6db13ad5655, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/A/010d9b792c054cd8b0aefc3ab09c2fbf, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/A/bcc01fb81baa40eda70649ba9b89ad82] into tmpdir=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/.tmp, totalSize=35.3 K 2024-11-27T12:25:19,816 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.Compactor(225): Compacting cfdf1edf206c48839298e7083fdec231, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1732710317321 2024-11-27T12:25:19,816 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] compactions.Compactor(225): Compacting 67e8e65a784045389118c6db13ad5655, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1732710317321 2024-11-27T12:25:19,817 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.Compactor(225): Compacting 842a0a725343457b9486270b91a3f6c2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1732710317505 2024-11-27T12:25:19,817 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] compactions.Compactor(225): Compacting 010d9b792c054cd8b0aefc3ab09c2fbf, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1732710317505 2024-11-27T12:25:19,817 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.Compactor(225): Compacting 1126a40417a341cbb1db114f53cbb579, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1732710318168 2024-11-27T12:25:19,817 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] compactions.Compactor(225): Compacting bcc01fb81baa40eda70649ba9b89ad82, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1732710318168 2024-11-27T12:25:19,825 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3c3055e4b5609cf4625dfa6b13256b83#B#compaction#204 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T12:25:19,825 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/.tmp/B/15cf97c1fa2a409b800d883ae16b1550 is 50, key is test_row_0/B:col10/1732710319309/Put/seqid=0 2024-11-27T12:25:19,830 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3c3055e4b5609cf4625dfa6b13256b83#A#compaction#205 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T12:25:19,830 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/.tmp/A/fb16592d6327417e81a6559188d7da16 is 50, key is test_row_0/A:col10/1732710319309/Put/seqid=0 2024-11-27T12:25:19,838 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742075_1251 (size=12207) 2024-11-27T12:25:19,843 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-27T12:25:19,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83. 2024-11-27T12:25:19,844 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2902): Flushing 3c3055e4b5609cf4625dfa6b13256b83 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-27T12:25:19,853 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/.tmp/B/15cf97c1fa2a409b800d883ae16b1550 as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/B/15cf97c1fa2a409b800d883ae16b1550 2024-11-27T12:25:19,860 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3c3055e4b5609cf4625dfa6b13256b83/B of 3c3055e4b5609cf4625dfa6b13256b83 into 15cf97c1fa2a409b800d883ae16b1550(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T12:25:19,860 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3c3055e4b5609cf4625dfa6b13256b83: 2024-11-27T12:25:19,860 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., storeName=3c3055e4b5609cf4625dfa6b13256b83/B, priority=13, startTime=1732710319814; duration=0sec 2024-11-27T12:25:19,860 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T12:25:19,860 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3c3055e4b5609cf4625dfa6b13256b83:B 2024-11-27T12:25:19,860 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T12:25:19,862 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T12:25:19,862 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HStore(1541): 3c3055e4b5609cf4625dfa6b13256b83/C is initiating minor compaction (all files) 2024-11-27T12:25:19,862 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3c3055e4b5609cf4625dfa6b13256b83/C in TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83. 2024-11-27T12:25:19,862 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/C/06c9741688a7405884a3aa541a3226da, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/C/6081cd35ef12471787e429634907855f, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/C/ed0015bbefa8428fb4ccd28e6859ebe7] into tmpdir=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/.tmp, totalSize=35.3 K 2024-11-27T12:25:19,863 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.Compactor(225): Compacting 06c9741688a7405884a3aa541a3226da, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1732710317321 2024-11-27T12:25:19,863 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.Compactor(225): Compacting 6081cd35ef12471787e429634907855f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1732710317505 2024-11-27T12:25:19,863 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.Compactor(225): Compacting ed0015bbefa8428fb4ccd28e6859ebe7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1732710318168 2024-11-27T12:25:19,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/.tmp/A/da7d72b097b54842acb90c5aa65fcbe6 is 50, key is test_row_0/A:col10/1732710319331/Put/seqid=0 2024-11-27T12:25:19,886 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742076_1252 (size=12207) 2024-11-27T12:25:19,890 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3c3055e4b5609cf4625dfa6b13256b83#C#compaction#207 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T12:25:19,890 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/.tmp/C/53394c9af01d4d5d9991e9c8d3aca180 is 50, key is test_row_0/C:col10/1732710319309/Put/seqid=0 2024-11-27T12:25:19,898 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/.tmp/A/fb16592d6327417e81a6559188d7da16 as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/A/fb16592d6327417e81a6559188d7da16 2024-11-27T12:25:19,904 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3c3055e4b5609cf4625dfa6b13256b83/A of 3c3055e4b5609cf4625dfa6b13256b83 into fb16592d6327417e81a6559188d7da16(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T12:25:19,904 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3c3055e4b5609cf4625dfa6b13256b83: 2024-11-27T12:25:19,904 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., storeName=3c3055e4b5609cf4625dfa6b13256b83/A, priority=13, startTime=1732710319814; duration=0sec 2024-11-27T12:25:19,904 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T12:25:19,904 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3c3055e4b5609cf4625dfa6b13256b83:A 2024-11-27T12:25:19,938 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742077_1253 (size=12001) 2024-11-27T12:25:19,940 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=113 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/.tmp/A/da7d72b097b54842acb90c5aa65fcbe6 2024-11-27T12:25:19,963 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83. as already flushing 2024-11-27T12:25:19,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(8855): Flush requested on 3c3055e4b5609cf4625dfa6b13256b83 2024-11-27T12:25:19,965 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742078_1254 (size=12207) 2024-11-27T12:25:19,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/.tmp/B/807bc563baf14cc485931c49c3666754 is 50, key is test_row_0/B:col10/1732710319331/Put/seqid=0 2024-11-27T12:25:19,977 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/.tmp/C/53394c9af01d4d5d9991e9c8d3aca180 as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/C/53394c9af01d4d5d9991e9c8d3aca180 2024-11-27T12:25:19,982 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3c3055e4b5609cf4625dfa6b13256b83/C of 3c3055e4b5609cf4625dfa6b13256b83 into 53394c9af01d4d5d9991e9c8d3aca180(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T12:25:19,982 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3c3055e4b5609cf4625dfa6b13256b83: 2024-11-27T12:25:19,982 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., storeName=3c3055e4b5609cf4625dfa6b13256b83/C, priority=13, startTime=1732710319814; duration=0sec 2024-11-27T12:25:19,982 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T12:25:19,982 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3c3055e4b5609cf4625dfa6b13256b83:C 2024-11-27T12:25:19,989 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742079_1255 (size=12001) 2024-11-27T12:25:19,993 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=113 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/.tmp/B/807bc563baf14cc485931c49c3666754 2024-11-27T12:25:20,005 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:20,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38952 deadline: 1732710379993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:20,005 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:20,005 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:20,005 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:20,007 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:20,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38986 deadline: 1732710379995, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:20,007 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:20,007 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:20,007 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:20,008 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:20,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38944 deadline: 1732710379997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:20,008 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:20,008 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:20,008 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:20,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-27T12:25:20,012 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:20,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38902 deadline: 1732710380005, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:20,012 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:20,012 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:20,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38978 deadline: 1732710380005, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:20,012 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:20,012 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:20,012 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:20,013 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:20,013 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:20,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/.tmp/C/7e6e262460e24e09888db376683306fc is 50, key is test_row_0/C:col10/1732710319331/Put/seqid=0 2024-11-27T12:25:20,066 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742080_1256 (size=12001) 2024-11-27T12:25:20,111 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:20,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38952 deadline: 1732710380108, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:20,112 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:20,112 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:20,112 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:20,119 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:20,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38944 deadline: 1732710380118, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:20,120 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:20,120 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:20,120 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:20,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38902 deadline: 1732710380118, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:20,120 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:20,120 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:20,120 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:20,120 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:20,120 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:20,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38986 deadline: 1732710380118, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:20,121 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:20,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38978 deadline: 1732710380118, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:20,121 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:20,121 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:20,121 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:20,122 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:20,122 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:20,122 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:20,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-27T12:25:20,320 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:20,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38952 deadline: 1732710380318, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:20,321 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:20,321 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:20,321 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:20,330 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:20,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38944 deadline: 1732710380328, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:20,331 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:20,331 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:20,330 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:20,331 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:20,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38902 deadline: 1732710380328, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:20,331 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:20,331 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:20,331 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:20,331 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:20,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38986 deadline: 1732710380329, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:20,331 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:20,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38978 deadline: 1732710380329, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:20,332 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:20,332 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:20,332 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:20,332 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:20,332 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:20,332 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:20,467 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=113 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/.tmp/C/7e6e262460e24e09888db376683306fc 2024-11-27T12:25:20,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/.tmp/A/da7d72b097b54842acb90c5aa65fcbe6 as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/A/da7d72b097b54842acb90c5aa65fcbe6 2024-11-27T12:25:20,474 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/A/da7d72b097b54842acb90c5aa65fcbe6, entries=150, sequenceid=113, filesize=11.7 K 2024-11-27T12:25:20,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/.tmp/B/807bc563baf14cc485931c49c3666754 as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/B/807bc563baf14cc485931c49c3666754 2024-11-27T12:25:20,479 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/B/807bc563baf14cc485931c49c3666754, entries=150, sequenceid=113, filesize=11.7 K 2024-11-27T12:25:20,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/.tmp/C/7e6e262460e24e09888db376683306fc as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/C/7e6e262460e24e09888db376683306fc 2024-11-27T12:25:20,492 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/C/7e6e262460e24e09888db376683306fc, entries=150, sequenceid=113, filesize=11.7 K 2024-11-27T12:25:20,493 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=100.63 KB/103050 for 3c3055e4b5609cf4625dfa6b13256b83 in 649ms, sequenceid=113, compaction requested=false 2024-11-27T12:25:20,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2603): Flush status journal for 3c3055e4b5609cf4625dfa6b13256b83: 2024-11-27T12:25:20,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83. 2024-11-27T12:25:20,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=67 2024-11-27T12:25:20,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36215 {}] master.HMaster(4169): Remote procedure done, pid=67 2024-11-27T12:25:20,496 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=67, resume processing ppid=66 2024-11-27T12:25:20,497 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=67, ppid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 804 msec 2024-11-27T12:25:20,498 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees in 809 msec 2024-11-27T12:25:20,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(8855): Flush requested on 3c3055e4b5609cf4625dfa6b13256b83 2024-11-27T12:25:20,630 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T12:25:20,630 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3c3055e4b5609cf4625dfa6b13256b83 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-11-27T12:25:20,635 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/.tmp/A/e441396b293b4eb6bb49b597fe03adc0 is 50, key is test_row_0/A:col10/1732710320629/Put/seqid=0 2024-11-27T12:25:20,640 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742081_1257 (size=14491) 2024-11-27T12:25:20,640 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=135 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/.tmp/A/e441396b293b4eb6bb49b597fe03adc0 2024-11-27T12:25:20,647 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/.tmp/B/e92a1c734b6e457797be0c0eecf7b586 is 50, key is test_row_0/B:col10/1732710320629/Put/seqid=0 2024-11-27T12:25:20,653 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742082_1258 (size=12151) 2024-11-27T12:25:20,654 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=135 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/.tmp/B/e92a1c734b6e457797be0c0eecf7b586 2024-11-27T12:25:20,658 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:20,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38902 deadline: 1732710380652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:20,659 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:20,659 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:20,659 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:20,659 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:20,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38952 deadline: 1732710380653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:20,659 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:20,659 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:20,659 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:20,661 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/.tmp/C/4e8dd2abfd6c477694e660cf59e81595 is 50, key is test_row_0/C:col10/1732710320629/Put/seqid=0 2024-11-27T12:25:20,663 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:20,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38978 deadline: 1732710380658, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:20,663 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:20,663 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:20,664 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:20,664 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:20,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38944 deadline: 1732710380658, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:20,664 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:20,664 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:20,664 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:20,664 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:20,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38986 deadline: 1732710380660, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:20,664 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:20,664 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:20,664 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:20,665 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742083_1259 (size=12151) 2024-11-27T12:25:20,771 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:20,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38902 deadline: 1732710380769, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:20,772 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:20,772 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:20,772 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:20,772 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:20,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38978 deadline: 1732710380769, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:20,772 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:20,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38944 deadline: 1732710380769, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:20,772 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:20,772 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:20,772 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:20,772 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:20,772 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:20,772 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:20,772 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:20,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38952 deadline: 1732710380769, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:20,772 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:20,773 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:20,773 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:20,773 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:20,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38986 deadline: 1732710380769, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:20,773 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:20,773 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:20,773 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:20,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-27T12:25:20,829 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T12:25:20,829 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36215 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-27T12:25:20,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36215 {}] procedure2.ProcedureExecutor(1139): Stored pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees 2024-11-27T12:25:20,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-27T12:25:20,831 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T12:25:20,832 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T12:25:20,832 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=69, ppid=68, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T12:25:20,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-27T12:25:20,981 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:20,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38902 deadline: 1732710380978, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:20,982 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:20,982 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:20,982 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:20,982 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:20,982 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:20,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38978 deadline: 1732710380978, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:20,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38944 deadline: 1732710380979, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:20,982 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:20,982 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:20,982 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:20,982 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:20,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38952 deadline: 1732710380979, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:20,982 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:20,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38986 deadline: 1732710380979, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:20,982 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:20,982 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:20,983 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:20,983 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:20,983 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:20,983 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:20,983 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:20,983 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:20,983 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:20,985 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-27T12:25:20,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83. 2024-11-27T12:25:20,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83. as already flushing 2024-11-27T12:25:20,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83. 2024-11-27T12:25:20,985 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 3c3055e4b5609cf4625dfa6b13256b83, NAME => 'TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:25:20,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 3c3055e4b5609cf4625dfa6b13256b83, NAME => 'TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:25:20,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36215 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3c3055e4b5609cf4625dfa6b13256b83, NAME => 'TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3c3055e4b5609cf4625dfa6b13256b83, NAME => 'TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:25:21,066 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=135 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/.tmp/C/4e8dd2abfd6c477694e660cf59e81595 2024-11-27T12:25:21,073 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/.tmp/A/e441396b293b4eb6bb49b597fe03adc0 as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/A/e441396b293b4eb6bb49b597fe03adc0 2024-11-27T12:25:21,078 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/A/e441396b293b4eb6bb49b597fe03adc0, entries=200, sequenceid=135, filesize=14.2 K 2024-11-27T12:25:21,079 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/.tmp/B/e92a1c734b6e457797be0c0eecf7b586 as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/B/e92a1c734b6e457797be0c0eecf7b586 2024-11-27T12:25:21,086 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/B/e92a1c734b6e457797be0c0eecf7b586, entries=150, sequenceid=135, filesize=11.9 K 2024-11-27T12:25:21,086 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/.tmp/C/4e8dd2abfd6c477694e660cf59e81595 as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/C/4e8dd2abfd6c477694e660cf59e81595 2024-11-27T12:25:21,093 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/C/4e8dd2abfd6c477694e660cf59e81595, entries=150, sequenceid=135, filesize=11.9 K 2024-11-27T12:25:21,094 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=100.63 KB/103050 for 3c3055e4b5609cf4625dfa6b13256b83 in 464ms, sequenceid=135, compaction requested=true 2024-11-27T12:25:21,094 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3c3055e4b5609cf4625dfa6b13256b83: 2024-11-27T12:25:21,094 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3c3055e4b5609cf4625dfa6b13256b83:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T12:25:21,094 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T12:25:21,094 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T12:25:21,094 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3c3055e4b5609cf4625dfa6b13256b83:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T12:25:21,094 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T12:25:21,094 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T12:25:21,094 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3c3055e4b5609cf4625dfa6b13256b83:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T12:25:21,094 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T12:25:21,095 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38699 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T12:25:21,095 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36359 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T12:25:21,095 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HStore(1541): 3c3055e4b5609cf4625dfa6b13256b83/A is initiating minor compaction (all files) 2024-11-27T12:25:21,095 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HStore(1541): 3c3055e4b5609cf4625dfa6b13256b83/B is initiating minor compaction (all files) 2024-11-27T12:25:21,095 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3c3055e4b5609cf4625dfa6b13256b83/B in TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83. 2024-11-27T12:25:21,095 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3c3055e4b5609cf4625dfa6b13256b83/A in TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83. 2024-11-27T12:25:21,095 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/A/fb16592d6327417e81a6559188d7da16, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/A/da7d72b097b54842acb90c5aa65fcbe6, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/A/e441396b293b4eb6bb49b597fe03adc0] into tmpdir=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/.tmp, totalSize=37.8 K 2024-11-27T12:25:21,095 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/B/15cf97c1fa2a409b800d883ae16b1550, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/B/807bc563baf14cc485931c49c3666754, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/B/e92a1c734b6e457797be0c0eecf7b586] into tmpdir=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/.tmp, totalSize=35.5 K 2024-11-27T12:25:21,096 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.Compactor(225): Compacting 15cf97c1fa2a409b800d883ae16b1550, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1732710318168 2024-11-27T12:25:21,096 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] compactions.Compactor(225): Compacting fb16592d6327417e81a6559188d7da16, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1732710318168 2024-11-27T12:25:21,096 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.Compactor(225): Compacting 807bc563baf14cc485931c49c3666754, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=113, earliestPutTs=1732710319327 2024-11-27T12:25:21,096 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] compactions.Compactor(225): Compacting da7d72b097b54842acb90c5aa65fcbe6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=113, earliestPutTs=1732710319327 2024-11-27T12:25:21,097 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.Compactor(225): Compacting e92a1c734b6e457797be0c0eecf7b586, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1732710319994 2024-11-27T12:25:21,097 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] compactions.Compactor(225): Compacting e441396b293b4eb6bb49b597fe03adc0, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1732710319978 2024-11-27T12:25:21,105 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3c3055e4b5609cf4625dfa6b13256b83#A#compaction#213 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-27T12:25:21,106 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/.tmp/A/032b5a7a90cb4aa2b41393bf6dd41061 is 50, key is test_row_0/A:col10/1732710320629/Put/seqid=0 2024-11-27T12:25:21,106 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3c3055e4b5609cf4625dfa6b13256b83#B#compaction#214 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T12:25:21,107 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/.tmp/B/4cff65fc94e14a13a380bd4c81d4fb1a is 50, key is test_row_0/B:col10/1732710320629/Put/seqid=0 2024-11-27T12:25:21,134 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742084_1260 (size=12459) 2024-11-27T12:25:21,138 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-27T12:25:21,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83. 2024-11-27T12:25:21,138 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2902): Flushing 3c3055e4b5609cf4625dfa6b13256b83 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-11-27T12:25:21,145 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/.tmp/B/4cff65fc94e14a13a380bd4c81d4fb1a as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/B/4cff65fc94e14a13a380bd4c81d4fb1a 2024-11-27T12:25:21,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-27T12:25:21,148 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742085_1261 (size=12459) 2024-11-27T12:25:21,154 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/.tmp/A/032b5a7a90cb4aa2b41393bf6dd41061 as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/A/032b5a7a90cb4aa2b41393bf6dd41061 2024-11-27T12:25:21,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/.tmp/A/82632257982c45a99f1b5235f8c8252e is 50, key is test_row_0/A:col10/1732710320656/Put/seqid=0 2024-11-27T12:25:21,166 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3c3055e4b5609cf4625dfa6b13256b83/A of 3c3055e4b5609cf4625dfa6b13256b83 into 032b5a7a90cb4aa2b41393bf6dd41061(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T12:25:21,166 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3c3055e4b5609cf4625dfa6b13256b83: 2024-11-27T12:25:21,166 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., storeName=3c3055e4b5609cf4625dfa6b13256b83/A, priority=13, startTime=1732710321094; duration=0sec 2024-11-27T12:25:21,166 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T12:25:21,166 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3c3055e4b5609cf4625dfa6b13256b83:A 2024-11-27T12:25:21,166 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T12:25:21,168 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36359 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T12:25:21,168 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HStore(1541): 3c3055e4b5609cf4625dfa6b13256b83/C is initiating minor compaction (all files) 2024-11-27T12:25:21,168 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3c3055e4b5609cf4625dfa6b13256b83/C in TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83. 2024-11-27T12:25:21,168 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/C/53394c9af01d4d5d9991e9c8d3aca180, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/C/7e6e262460e24e09888db376683306fc, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/C/4e8dd2abfd6c477694e660cf59e81595] into tmpdir=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/.tmp, totalSize=35.5 K 2024-11-27T12:25:21,169 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] compactions.Compactor(225): Compacting 53394c9af01d4d5d9991e9c8d3aca180, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1732710318168 2024-11-27T12:25:21,169 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7e6e262460e24e09888db376683306fc, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=113, earliestPutTs=1732710319327 2024-11-27T12:25:21,170 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4e8dd2abfd6c477694e660cf59e81595, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1732710319994 2024-11-27T12:25:21,180 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3c3055e4b5609cf4625dfa6b13256b83/B of 3c3055e4b5609cf4625dfa6b13256b83 into 4cff65fc94e14a13a380bd4c81d4fb1a(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T12:25:21,180 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3c3055e4b5609cf4625dfa6b13256b83: 2024-11-27T12:25:21,180 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., storeName=3c3055e4b5609cf4625dfa6b13256b83/B, priority=13, startTime=1732710321094; duration=0sec 2024-11-27T12:25:21,180 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T12:25:21,181 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3c3055e4b5609cf4625dfa6b13256b83:B 2024-11-27T12:25:21,185 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3c3055e4b5609cf4625dfa6b13256b83#C#compaction#216 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T12:25:21,186 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/.tmp/C/03aec8ffbd7244eda7166f699f133df3 is 50, key is test_row_0/C:col10/1732710320629/Put/seqid=0 2024-11-27T12:25:21,190 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742086_1262 (size=12151) 2024-11-27T12:25:21,197 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=153 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/.tmp/A/82632257982c45a99f1b5235f8c8252e 2024-11-27T12:25:21,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/.tmp/B/ac573ba6a42d487185d30bd8e5881aef is 50, key is test_row_0/B:col10/1732710320656/Put/seqid=0 2024-11-27T12:25:21,246 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742087_1263 (size=12459) 2024-11-27T12:25:21,251 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742088_1264 (size=12151) 2024-11-27T12:25:21,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(8855): Flush requested on 3c3055e4b5609cf4625dfa6b13256b83 2024-11-27T12:25:21,291 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83. as already flushing 2024-11-27T12:25:21,308 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:21,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38902 deadline: 1732710381304, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:21,309 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:21,309 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:21,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38978 deadline: 1732710381305, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:21,309 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:21,309 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:21,309 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:21,309 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:21,309 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:21,312 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:21,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38944 deadline: 1732710381307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:21,312 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:21,312 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:21,312 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:21,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38986 deadline: 1732710381308, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:21,312 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:21,312 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:21,313 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:21,312 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:21,313 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:21,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38952 deadline: 1732710381309, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:21,313 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:21,313 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:21,313 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:21,420 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:21,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38902 deadline: 1732710381418, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:21,420 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:21,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38978 deadline: 1732710381418, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:21,420 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:21,420 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:21,420 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:21,420 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:21,420 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:21,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38944 deadline: 1732710381418, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:21,421 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:21,421 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:21,421 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:21,421 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:21,421 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:21,421 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:21,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38986 deadline: 1732710381418, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:21,421 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:21,421 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:21,421 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:21,421 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:21,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38952 deadline: 1732710381418, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:21,421 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:21,421 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:21,421 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:21,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-27T12:25:21,628 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:21,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38944 deadline: 1732710381628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:21,629 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:21,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38978 deadline: 1732710381628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:21,629 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:21,629 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:21,629 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:21,629 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:21,629 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:21,629 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:21,629 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:21,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38986 deadline: 1732710381629, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:21,630 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:21,630 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:21,630 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:21,630 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:21,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38902 deadline: 1732710381629, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:21,630 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:21,630 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:21,630 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:21,630 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:21,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38952 deadline: 1732710381629, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:21,630 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:21,630 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:21,630 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:21,652 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=153 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/.tmp/B/ac573ba6a42d487185d30bd8e5881aef 2024-11-27T12:25:21,656 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/.tmp/C/03aec8ffbd7244eda7166f699f133df3 as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/C/03aec8ffbd7244eda7166f699f133df3 2024-11-27T12:25:21,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/.tmp/C/e9848e3c8ca948aaa54f243144497bca is 50, key is test_row_0/C:col10/1732710320656/Put/seqid=0 2024-11-27T12:25:21,663 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3c3055e4b5609cf4625dfa6b13256b83/C of 3c3055e4b5609cf4625dfa6b13256b83 into 03aec8ffbd7244eda7166f699f133df3(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T12:25:21,663 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3c3055e4b5609cf4625dfa6b13256b83: 2024-11-27T12:25:21,663 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., storeName=3c3055e4b5609cf4625dfa6b13256b83/C, priority=13, startTime=1732710321094; duration=0sec 2024-11-27T12:25:21,663 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T12:25:21,663 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3c3055e4b5609cf4625dfa6b13256b83:C 2024-11-27T12:25:21,683 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742089_1265 (size=12151) 2024-11-27T12:25:21,940 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:21,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38944 deadline: 1732710381938, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:21,940 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:21,940 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:21,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38978 deadline: 1732710381938, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:21,940 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:21,940 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:21,940 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:21,940 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:21,940 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:21,940 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:21,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38986 deadline: 1732710381938, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:21,941 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:21,941 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:21,941 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:21,941 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:21,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38902 deadline: 1732710381938, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:21,941 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:21,941 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:21,941 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:21,941 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:21,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38952 deadline: 1732710381938, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:21,941 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:21,941 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:21,941 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:21,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-27T12:25:22,084 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=153 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/.tmp/C/e9848e3c8ca948aaa54f243144497bca 2024-11-27T12:25:22,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/.tmp/A/82632257982c45a99f1b5235f8c8252e as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/A/82632257982c45a99f1b5235f8c8252e 2024-11-27T12:25:22,099 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/A/82632257982c45a99f1b5235f8c8252e, entries=150, sequenceid=153, filesize=11.9 K 2024-11-27T12:25:22,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/.tmp/B/ac573ba6a42d487185d30bd8e5881aef as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/B/ac573ba6a42d487185d30bd8e5881aef 2024-11-27T12:25:22,107 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/B/ac573ba6a42d487185d30bd8e5881aef, entries=150, sequenceid=153, filesize=11.9 K 2024-11-27T12:25:22,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/.tmp/C/e9848e3c8ca948aaa54f243144497bca as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/C/e9848e3c8ca948aaa54f243144497bca 2024-11-27T12:25:22,113 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/C/e9848e3c8ca948aaa54f243144497bca, entries=150, sequenceid=153, filesize=11.9 K 2024-11-27T12:25:22,113 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=100.63 KB/103050 for 3c3055e4b5609cf4625dfa6b13256b83 in 975ms, sequenceid=153, compaction requested=false 2024-11-27T12:25:22,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2603): Flush status journal for 3c3055e4b5609cf4625dfa6b13256b83: 2024-11-27T12:25:22,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83. 2024-11-27T12:25:22,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=69 2024-11-27T12:25:22,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36215 {}] master.HMaster(4169): Remote procedure done, pid=69 2024-11-27T12:25:22,117 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=69, resume processing ppid=68 2024-11-27T12:25:22,117 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=69, ppid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2830 sec 2024-11-27T12:25:22,120 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees in 1.2890 sec 2024-11-27T12:25:22,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(8855): Flush requested on 3c3055e4b5609cf4625dfa6b13256b83 2024-11-27T12:25:22,450 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T12:25:22,450 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3c3055e4b5609cf4625dfa6b13256b83 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-11-27T12:25:22,455 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/.tmp/A/3862eb8dfd194caebcfa400f51b8b83a is 50, key is test_row_0/A:col10/1732710321303/Put/seqid=0 2024-11-27T12:25:22,459 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742090_1266 (size=12151) 2024-11-27T12:25:22,467 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:22,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38944 deadline: 1732710382462, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:22,467 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:22,467 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:22,467 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:22,467 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:22,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38978 deadline: 1732710382463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:22,468 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:22,468 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:22,468 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:22,468 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:22,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38986 deadline: 1732710382463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:22,468 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:22,468 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:22,468 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:22,468 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:22,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38902 deadline: 1732710382463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:22,468 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:22,468 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:22,468 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:22,468 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:22,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38952 deadline: 1732710382464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:22,469 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:22,469 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:22,469 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:22,581 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:22,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38944 deadline: 1732710382578, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:22,581 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:22,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38978 deadline: 1732710382578, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:22,581 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:22,581 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:22,582 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:22,582 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:22,582 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:22,582 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:22,582 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:22,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38986 deadline: 1732710382578, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:22,582 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:22,582 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:22,582 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:22,582 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:22,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38902 deadline: 1732710382578, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:22,582 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:22,582 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:22,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38952 deadline: 1732710382578, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:22,582 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:22,582 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:22,582 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:22,583 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:22,583 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:22,791 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:22,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38944 deadline: 1732710382788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:22,791 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:22,792 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:22,791 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:22,792 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:22,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38978 deadline: 1732710382789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:22,792 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:22,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38986 deadline: 1732710382789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:22,792 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:22,792 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:22,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38902 deadline: 1732710382789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:22,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38952 deadline: 1732710382789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:22,792 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:22,793 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:22,793 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:22,793 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:22,793 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:22,793 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:22,793 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:22,793 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:22,793 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:22,793 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:22,794 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:22,794 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:22,860 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=177 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/.tmp/A/3862eb8dfd194caebcfa400f51b8b83a 2024-11-27T12:25:22,870 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/.tmp/B/a8e383a16fb6432498bfc82cd4c3539e is 50, key is test_row_0/B:col10/1732710321303/Put/seqid=0 2024-11-27T12:25:22,882 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742091_1267 (size=12151) 2024-11-27T12:25:22,883 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=177 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/.tmp/B/a8e383a16fb6432498bfc82cd4c3539e 2024-11-27T12:25:22,906 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/.tmp/C/7e54f38d993b4e05b78daece4339348f is 50, key is test_row_0/C:col10/1732710321303/Put/seqid=0 2024-11-27T12:25:22,912 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742092_1268 (size=12151) 2024-11-27T12:25:22,915 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=177 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/.tmp/C/7e54f38d993b4e05b78daece4339348f 2024-11-27T12:25:22,925 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/.tmp/A/3862eb8dfd194caebcfa400f51b8b83a as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/A/3862eb8dfd194caebcfa400f51b8b83a 2024-11-27T12:25:22,932 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/A/3862eb8dfd194caebcfa400f51b8b83a, entries=150, sequenceid=177, filesize=11.9 K 2024-11-27T12:25:22,933 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/.tmp/B/a8e383a16fb6432498bfc82cd4c3539e as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/B/a8e383a16fb6432498bfc82cd4c3539e 2024-11-27T12:25:22,940 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/B/a8e383a16fb6432498bfc82cd4c3539e, entries=150, sequenceid=177, filesize=11.9 K 2024-11-27T12:25:22,943 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/.tmp/C/7e54f38d993b4e05b78daece4339348f as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/C/7e54f38d993b4e05b78daece4339348f 2024-11-27T12:25:22,950 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/C/7e54f38d993b4e05b78daece4339348f, entries=150, sequenceid=177, filesize=11.9 K 2024-11-27T12:25:22,951 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=80.51 KB/82440 for 3c3055e4b5609cf4625dfa6b13256b83 in 501ms, sequenceid=177, compaction requested=true 2024-11-27T12:25:22,951 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3c3055e4b5609cf4625dfa6b13256b83: 2024-11-27T12:25:22,951 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3c3055e4b5609cf4625dfa6b13256b83:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T12:25:22,951 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T12:25:22,951 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T12:25:22,951 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3c3055e4b5609cf4625dfa6b13256b83:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T12:25:22,951 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T12:25:22,951 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T12:25:22,951 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3c3055e4b5609cf4625dfa6b13256b83:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T12:25:22,951 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T12:25:22,953 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T12:25:22,953 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T12:25:22,953 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HStore(1541): 3c3055e4b5609cf4625dfa6b13256b83/A is initiating minor compaction (all files) 2024-11-27T12:25:22,953 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HStore(1541): 3c3055e4b5609cf4625dfa6b13256b83/B is initiating minor compaction (all files) 2024-11-27T12:25:22,953 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3c3055e4b5609cf4625dfa6b13256b83/A in TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83. 2024-11-27T12:25:22,953 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3c3055e4b5609cf4625dfa6b13256b83/B in TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83. 2024-11-27T12:25:22,953 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/B/4cff65fc94e14a13a380bd4c81d4fb1a, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/B/ac573ba6a42d487185d30bd8e5881aef, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/B/a8e383a16fb6432498bfc82cd4c3539e] into tmpdir=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/.tmp, totalSize=35.9 K 2024-11-27T12:25:22,953 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/A/032b5a7a90cb4aa2b41393bf6dd41061, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/A/82632257982c45a99f1b5235f8c8252e, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/A/3862eb8dfd194caebcfa400f51b8b83a] into tmpdir=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/.tmp, totalSize=35.9 K 2024-11-27T12:25:22,954 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.Compactor(225): Compacting 4cff65fc94e14a13a380bd4c81d4fb1a, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1732710319994 2024-11-27T12:25:22,954 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] compactions.Compactor(225): Compacting 032b5a7a90cb4aa2b41393bf6dd41061, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1732710319994 2024-11-27T12:25:22,954 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.Compactor(225): Compacting ac573ba6a42d487185d30bd8e5881aef, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=153, earliestPutTs=1732710320656 2024-11-27T12:25:22,954 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] compactions.Compactor(225): Compacting 82632257982c45a99f1b5235f8c8252e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=153, earliestPutTs=1732710320656 2024-11-27T12:25:22,954 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.Compactor(225): Compacting a8e383a16fb6432498bfc82cd4c3539e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=177, earliestPutTs=1732710321303 2024-11-27T12:25:22,955 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3862eb8dfd194caebcfa400f51b8b83a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=177, earliestPutTs=1732710321303 2024-11-27T12:25:22,963 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3c3055e4b5609cf4625dfa6b13256b83#A#compaction#222 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T12:25:22,963 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/.tmp/A/0bed16d005ea457f849e554a21ba34d3 is 50, key is test_row_0/A:col10/1732710321303/Put/seqid=0 2024-11-27T12:25:22,966 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3c3055e4b5609cf4625dfa6b13256b83#B#compaction#223 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T12:25:22,967 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/.tmp/B/b4b29f12a9c04d66b837557a085f8c0d is 50, key is test_row_0/B:col10/1732710321303/Put/seqid=0 2024-11-27T12:25:22,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-27T12:25:22,978 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T12:25:22,978 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36215 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-27T12:25:22,979 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742093_1269 (size=12561) 2024-11-27T12:25:22,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36215 {}] procedure2.ProcedureExecutor(1139): Stored pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees 2024-11-27T12:25:22,981 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T12:25:22,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-27T12:25:22,982 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T12:25:22,982 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=71, ppid=70, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T12:25:22,988 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742094_1270 (size=12561) 2024-11-27T12:25:23,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-27T12:25:23,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(8855): Flush requested on 3c3055e4b5609cf4625dfa6b13256b83 2024-11-27T12:25:23,099 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T12:25:23,099 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3c3055e4b5609cf4625dfa6b13256b83 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-27T12:25:23,104 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/.tmp/A/bee2be8112594615bbc186c2d5260738 is 50, key is test_row_0/A:col10/1732710323098/Put/seqid=0 2024-11-27T12:25:23,109 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742095_1271 (size=12151) 2024-11-27T12:25:23,129 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:23,129 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:23,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38952 deadline: 1732710383122, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:23,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38944 deadline: 1732710383123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:23,130 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:23,130 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:23,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38902 deadline: 1732710383126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:23,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38978 deadline: 1732710383124, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:23,130 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:23,130 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:23,130 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:23,131 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:23,131 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:23,131 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:23,131 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:23,131 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:23,131 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:23,132 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:23,132 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:23,132 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:23,132 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:23,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38986 deadline: 1732710383127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:23,133 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:23,133 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:23,133 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:23,140 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-27T12:25:23,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83. 2024-11-27T12:25:23,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83. as already flushing 2024-11-27T12:25:23,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83. 2024-11-27T12:25:23,141 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 3c3055e4b5609cf4625dfa6b13256b83, NAME => 'TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:25:23,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 3c3055e4b5609cf4625dfa6b13256b83, NAME => 'TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:25:23,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36215 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3c3055e4b5609cf4625dfa6b13256b83, NAME => 'TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3c3055e4b5609cf4625dfa6b13256b83, NAME => 'TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:25:23,242 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:23,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38986 deadline: 1732710383239, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:23,242 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:23,242 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:23,243 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:23,243 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:23,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38952 deadline: 1732710383239, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:23,243 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:23,243 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:23,243 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:23,243 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:23,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38902 deadline: 1732710383239, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:23,243 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:23,243 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:23,243 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:23,243 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:23,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38978 deadline: 1732710383240, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:23,244 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:23,244 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:23,244 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:23,244 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:23,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38944 deadline: 1732710383240, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:23,244 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:23,244 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:23,244 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:23,294 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-27T12:25:23,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83. 2024-11-27T12:25:23,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83. as already flushing 2024-11-27T12:25:23,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83. 2024-11-27T12:25:23,295 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 3c3055e4b5609cf4625dfa6b13256b83, NAME => 'TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:25:23,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 3c3055e4b5609cf4625dfa6b13256b83, NAME => 'TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:25:23,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36215 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3c3055e4b5609cf4625dfa6b13256b83, NAME => 'TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3c3055e4b5609cf4625dfa6b13256b83, NAME => 'TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:25:23,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-27T12:25:23,384 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/.tmp/A/0bed16d005ea457f849e554a21ba34d3 as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/A/0bed16d005ea457f849e554a21ba34d3 2024-11-27T12:25:23,388 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3c3055e4b5609cf4625dfa6b13256b83/A of 3c3055e4b5609cf4625dfa6b13256b83 into 0bed16d005ea457f849e554a21ba34d3(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T12:25:23,388 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3c3055e4b5609cf4625dfa6b13256b83: 2024-11-27T12:25:23,388 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., storeName=3c3055e4b5609cf4625dfa6b13256b83/A, priority=13, startTime=1732710322951; duration=0sec 2024-11-27T12:25:23,388 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T12:25:23,388 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3c3055e4b5609cf4625dfa6b13256b83:A 2024-11-27T12:25:23,388 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T12:25:23,389 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T12:25:23,390 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HStore(1541): 3c3055e4b5609cf4625dfa6b13256b83/C is initiating minor compaction (all files) 2024-11-27T12:25:23,390 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3c3055e4b5609cf4625dfa6b13256b83/C in TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83. 2024-11-27T12:25:23,390 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/C/03aec8ffbd7244eda7166f699f133df3, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/C/e9848e3c8ca948aaa54f243144497bca, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/C/7e54f38d993b4e05b78daece4339348f] into tmpdir=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/.tmp, totalSize=35.9 K 2024-11-27T12:25:23,391 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] compactions.Compactor(225): Compacting 03aec8ffbd7244eda7166f699f133df3, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1732710319994 2024-11-27T12:25:23,392 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] compactions.Compactor(225): Compacting e9848e3c8ca948aaa54f243144497bca, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=153, earliestPutTs=1732710320656 2024-11-27T12:25:23,392 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7e54f38d993b4e05b78daece4339348f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=177, earliestPutTs=1732710321303 2024-11-27T12:25:23,393 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/.tmp/B/b4b29f12a9c04d66b837557a085f8c0d as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/B/b4b29f12a9c04d66b837557a085f8c0d 2024-11-27T12:25:23,402 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3c3055e4b5609cf4625dfa6b13256b83/B of 3c3055e4b5609cf4625dfa6b13256b83 into b4b29f12a9c04d66b837557a085f8c0d(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T12:25:23,402 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3c3055e4b5609cf4625dfa6b13256b83: 2024-11-27T12:25:23,402 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., storeName=3c3055e4b5609cf4625dfa6b13256b83/B, priority=13, startTime=1732710322951; duration=0sec 2024-11-27T12:25:23,402 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T12:25:23,402 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3c3055e4b5609cf4625dfa6b13256b83:B 2024-11-27T12:25:23,406 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3c3055e4b5609cf4625dfa6b13256b83#C#compaction#225 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T12:25:23,406 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/.tmp/C/fdc5ebc404ba4ff39280b4e5ef60cf65 is 50, key is test_row_0/C:col10/1732710321303/Put/seqid=0 2024-11-27T12:25:23,422 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742096_1272 (size=12561) 2024-11-27T12:25:23,427 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/.tmp/C/fdc5ebc404ba4ff39280b4e5ef60cf65 as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/C/fdc5ebc404ba4ff39280b4e5ef60cf65 2024-11-27T12:25:23,432 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3c3055e4b5609cf4625dfa6b13256b83/C of 3c3055e4b5609cf4625dfa6b13256b83 into fdc5ebc404ba4ff39280b4e5ef60cf65(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T12:25:23,432 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3c3055e4b5609cf4625dfa6b13256b83: 2024-11-27T12:25:23,432 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., storeName=3c3055e4b5609cf4625dfa6b13256b83/C, priority=13, startTime=1732710322951; duration=0sec 2024-11-27T12:25:23,432 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T12:25:23,432 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3c3055e4b5609cf4625dfa6b13256b83:C 2024-11-27T12:25:23,449 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:23,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38986 deadline: 1732710383448, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:23,450 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-27T12:25:23,450 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:23,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83. 2024-11-27T12:25:23,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38902 deadline: 1732710383448, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:23,450 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:23,450 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:23,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83. as already flushing 2024-11-27T12:25:23,450 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:23,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83. 2024-11-27T12:25:23,450 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:23,450 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:23,450 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 3c3055e4b5609cf4625dfa6b13256b83, NAME => 'TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:25:23,450 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:23,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 3c3055e4b5609cf4625dfa6b13256b83, NAME => 'TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:25:23,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36215 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3c3055e4b5609cf4625dfa6b13256b83, NAME => 'TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3c3055e4b5609cf4625dfa6b13256b83, NAME => 'TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:25:23,452 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:23,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38944 deadline: 1732710383449, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:23,452 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:23,452 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:23,452 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:23,452 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:23,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38952 deadline: 1732710383449, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:23,452 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:23,452 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:23,452 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:23,453 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:23,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38978 deadline: 1732710383449, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:23,453 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:23,453 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:23,453 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:23,510 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=193 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/.tmp/A/bee2be8112594615bbc186c2d5260738 2024-11-27T12:25:23,517 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/.tmp/B/e72b07f730c74167ab92afa7ab900b2e is 50, key is test_row_0/B:col10/1732710323098/Put/seqid=0 2024-11-27T12:25:23,535 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742097_1273 (size=12151) 2024-11-27T12:25:23,536 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=193 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/.tmp/B/e72b07f730c74167ab92afa7ab900b2e 2024-11-27T12:25:23,542 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/.tmp/C/96775b63a8f94aeabe40c656fdd1b0fa is 50, key is test_row_0/C:col10/1732710323098/Put/seqid=0 2024-11-27T12:25:23,557 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742098_1274 (size=12151) 2024-11-27T12:25:23,602 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-27T12:25:23,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83. 2024-11-27T12:25:23,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83. as already flushing 2024-11-27T12:25:23,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83. 2024-11-27T12:25:23,603 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 3c3055e4b5609cf4625dfa6b13256b83, NAME => 'TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:25:23,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 3c3055e4b5609cf4625dfa6b13256b83, NAME => 'TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:25:23,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36215 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3c3055e4b5609cf4625dfa6b13256b83, NAME => 'TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3c3055e4b5609cf4625dfa6b13256b83, NAME => 'TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:25:23,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-27T12:25:23,756 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-27T12:25:23,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83. 2024-11-27T12:25:23,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83. as already flushing 2024-11-27T12:25:23,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83. 2024-11-27T12:25:23,757 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 3c3055e4b5609cf4625dfa6b13256b83, NAME => 'TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:25:23,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 3c3055e4b5609cf4625dfa6b13256b83, NAME => 'TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:25:23,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36215 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3c3055e4b5609cf4625dfa6b13256b83, NAME => 'TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3c3055e4b5609cf4625dfa6b13256b83, NAME => 'TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:25:23,760 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:23,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38986 deadline: 1732710383758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:23,760 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:23,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38902 deadline: 1732710383759, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:23,761 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:23,761 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:23,761 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:23,761 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:23,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38952 deadline: 1732710383759, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:23,762 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:23,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38944 deadline: 1732710383759, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:23,762 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:23,762 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:23,762 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:23,762 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:23,762 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:23,762 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:23,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38978 deadline: 1732710383760, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:23,762 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:23,762 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:23,762 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:23,762 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:23,762 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:23,762 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:23,762 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:23,912 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-27T12:25:23,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83. 2024-11-27T12:25:23,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83. as already flushing 2024-11-27T12:25:23,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83. 2024-11-27T12:25:23,913 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 3c3055e4b5609cf4625dfa6b13256b83, NAME => 'TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:25:23,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 3c3055e4b5609cf4625dfa6b13256b83, NAME => 'TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:25:23,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36215 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3c3055e4b5609cf4625dfa6b13256b83, NAME => 'TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3c3055e4b5609cf4625dfa6b13256b83, NAME => 'TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:25:23,958 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=193 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/.tmp/C/96775b63a8f94aeabe40c656fdd1b0fa 2024-11-27T12:25:23,962 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/.tmp/A/bee2be8112594615bbc186c2d5260738 as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/A/bee2be8112594615bbc186c2d5260738 2024-11-27T12:25:23,966 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/A/bee2be8112594615bbc186c2d5260738, entries=150, sequenceid=193, filesize=11.9 K 2024-11-27T12:25:23,967 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/.tmp/B/e72b07f730c74167ab92afa7ab900b2e as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/B/e72b07f730c74167ab92afa7ab900b2e 2024-11-27T12:25:23,971 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/B/e72b07f730c74167ab92afa7ab900b2e, entries=150, sequenceid=193, filesize=11.9 K 2024-11-27T12:25:23,971 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/.tmp/C/96775b63a8f94aeabe40c656fdd1b0fa as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/C/96775b63a8f94aeabe40c656fdd1b0fa 2024-11-27T12:25:23,975 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/C/96775b63a8f94aeabe40c656fdd1b0fa, entries=150, sequenceid=193, filesize=11.9 K 2024-11-27T12:25:23,976 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=114.05 KB/116790 for 3c3055e4b5609cf4625dfa6b13256b83 in 877ms, sequenceid=193, compaction requested=false 2024-11-27T12:25:23,976 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3c3055e4b5609cf4625dfa6b13256b83: 2024-11-27T12:25:24,066 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-27T12:25:24,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83. 2024-11-27T12:25:24,067 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2902): Flushing 3c3055e4b5609cf4625dfa6b13256b83 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-27T12:25:24,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/.tmp/A/5b554eb6968b4e12b99cb76384381b1b is 50, key is test_row_0/A:col10/1732710323123/Put/seqid=0 2024-11-27T12:25:24,075 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742099_1275 (size=12151) 2024-11-27T12:25:24,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-27T12:25:24,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(8855): Flush requested on 3c3055e4b5609cf4625dfa6b13256b83 2024-11-27T12:25:24,270 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83. as already flushing 2024-11-27T12:25:24,288 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:24,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38978 deadline: 1732710384282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:24,288 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:24,288 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:24,288 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:24,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38986 deadline: 1732710384282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:24,288 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:24,289 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:24,289 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:24,289 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:24,291 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:24,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38902 deadline: 1732710384286, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:24,292 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:24,292 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:24,292 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:24,293 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:24,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38944 deadline: 1732710384287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:24,293 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:24,293 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:24,293 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:24,294 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:24,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38952 deadline: 1732710384289, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:24,294 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:24,294 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:24,294 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:24,400 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:24,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38986 deadline: 1732710384398, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:24,401 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:24,401 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:24,401 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:24,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38978 deadline: 1732710384398, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:24,401 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:24,401 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:24,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38944 deadline: 1732710384398, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:24,402 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:24,402 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:24,402 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:24,402 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:24,402 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:24,402 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:24,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38952 deadline: 1732710384398, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:24,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38902 deadline: 1732710384398, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:24,402 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:24,402 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:24,402 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:24,402 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:24,402 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:24,402 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:24,402 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:24,402 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:24,477 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=216 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/.tmp/A/5b554eb6968b4e12b99cb76384381b1b 2024-11-27T12:25:24,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/.tmp/B/6f72a4387d204bda9ce5046d5117891e is 50, key is test_row_0/B:col10/1732710323123/Put/seqid=0 2024-11-27T12:25:24,497 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742100_1276 (size=12151) 2024-11-27T12:25:24,498 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=216 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/.tmp/B/6f72a4387d204bda9ce5046d5117891e 2024-11-27T12:25:24,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/.tmp/C/328a92f8c4d9445ab7c91fbfcc76c670 is 50, key is test_row_0/C:col10/1732710323123/Put/seqid=0 2024-11-27T12:25:24,512 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742101_1277 (size=12151) 2024-11-27T12:25:24,611 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:24,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38978 deadline: 1732710384608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:24,612 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:24,612 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:24,612 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:24,612 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:24,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38986 deadline: 1732710384608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:24,612 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:24,612 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:24,612 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:24,612 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:24,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38944 deadline: 1732710384608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:24,612 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:24,613 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:24,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38952 deadline: 1732710384608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:24,613 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:24,613 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:24,613 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:24,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38902 deadline: 1732710384608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:24,613 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:24,613 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:24,613 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:24,614 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:24,614 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:24,614 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:24,913 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=216 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/.tmp/C/328a92f8c4d9445ab7c91fbfcc76c670 2024-11-27T12:25:24,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/.tmp/A/5b554eb6968b4e12b99cb76384381b1b as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/A/5b554eb6968b4e12b99cb76384381b1b 2024-11-27T12:25:24,921 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:24,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38986 deadline: 1732710384918, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:24,921 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:24,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38944 deadline: 1732710384918, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:24,921 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:24,921 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:24,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38952 deadline: 1732710384918, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:24,921 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:24,921 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:24,921 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:24,921 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:24,921 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:24,921 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:24,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38978 deadline: 1732710384918, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:24,922 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/A/5b554eb6968b4e12b99cb76384381b1b, entries=150, sequenceid=216, filesize=11.9 K 2024-11-27T12:25:24,922 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:24,922 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:24,922 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:24,922 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:24,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38902 deadline: 1732710384918, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:24,922 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:24,922 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:24,922 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:24,922 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:24,922 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:24,922 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:24,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/.tmp/B/6f72a4387d204bda9ce5046d5117891e as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/B/6f72a4387d204bda9ce5046d5117891e 2024-11-27T12:25:24,926 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/B/6f72a4387d204bda9ce5046d5117891e, entries=150, sequenceid=216, filesize=11.9 K 2024-11-27T12:25:24,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/.tmp/C/328a92f8c4d9445ab7c91fbfcc76c670 as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/C/328a92f8c4d9445ab7c91fbfcc76c670 2024-11-27T12:25:24,931 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/C/328a92f8c4d9445ab7c91fbfcc76c670, entries=150, sequenceid=216, filesize=11.9 K 2024-11-27T12:25:24,932 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=93.93 KB/96180 for 3c3055e4b5609cf4625dfa6b13256b83 in 865ms, sequenceid=216, compaction requested=true 2024-11-27T12:25:24,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2603): Flush status journal for 3c3055e4b5609cf4625dfa6b13256b83: 2024-11-27T12:25:24,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83. 2024-11-27T12:25:24,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=71 2024-11-27T12:25:24,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36215 {}] master.HMaster(4169): Remote procedure done, pid=71 2024-11-27T12:25:24,935 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=71, resume processing ppid=70 2024-11-27T12:25:24,935 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=71, ppid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9510 sec 2024-11-27T12:25:24,937 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees in 1.9570 sec 2024-11-27T12:25:25,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-27T12:25:25,128 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T12:25:25,129 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36215 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-27T12:25:25,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36215 {}] procedure2.ProcedureExecutor(1139): Stored pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees 2024-11-27T12:25:25,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-27T12:25:25,130 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T12:25:25,131 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T12:25:25,131 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=73, ppid=72, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T12:25:25,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-27T12:25:25,285 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-27T12:25:25,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83. 2024-11-27T12:25:25,285 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2902): Flushing 3c3055e4b5609cf4625dfa6b13256b83 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-11-27T12:25:25,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/.tmp/A/c493b74a85404fd5ac67218aceff4053 is 50, key is test_row_0/A:col10/1732710324286/Put/seqid=0 2024-11-27T12:25:25,293 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742102_1278 (size=12151) 2024-11-27T12:25:25,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(8855): Flush requested on 3c3055e4b5609cf4625dfa6b13256b83 2024-11-27T12:25:25,430 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83. as already flushing 2024-11-27T12:25:25,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-27T12:25:25,451 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:25,451 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:25,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38944 deadline: 1732710385446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:25,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38952 deadline: 1732710385446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:25,452 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:25,452 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:25,452 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:25,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38978 deadline: 1732710385447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:25,452 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:25,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38902 deadline: 1732710385447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:25,452 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:25,452 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:25,452 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:25,452 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:25,452 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:25,452 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:25,452 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:25,452 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:25,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38986 deadline: 1732710385448, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:25,452 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:25,452 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:25,452 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:25,452 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:25,452 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:25,452 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:25,562 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:25,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38944 deadline: 1732710385558, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:25,562 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:25,562 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:25,562 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:25,562 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:25,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38978 deadline: 1732710385558, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:25,562 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:25,562 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:25,562 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:25,562 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:25,562 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:25,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38902 deadline: 1732710385558, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:25,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38952 deadline: 1732710385558, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:25,563 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:25,563 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:25,563 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:25,563 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:25,563 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:25,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38986 deadline: 1732710385558, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:25,563 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:25,563 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:25,563 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:25,563 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:25,563 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:25,694 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=233 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/.tmp/A/c493b74a85404fd5ac67218aceff4053 2024-11-27T12:25:25,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/.tmp/B/0da8d57788a549c7a9e737ad8269a4cc is 50, key is test_row_0/B:col10/1732710324286/Put/seqid=0 2024-11-27T12:25:25,704 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742103_1279 (size=12151) 2024-11-27T12:25:25,705 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=233 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/.tmp/B/0da8d57788a549c7a9e737ad8269a4cc 2024-11-27T12:25:25,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/.tmp/C/63b7c461bfc840daa82a35801d6ddeec is 50, key is test_row_0/C:col10/1732710324286/Put/seqid=0 2024-11-27T12:25:25,714 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742104_1280 (size=12151) 2024-11-27T12:25:25,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-27T12:25:25,770 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:25,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38944 deadline: 1732710385768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:25,771 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:25,771 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:25,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38978 deadline: 1732710385768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:25,771 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:25,771 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:25,771 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:25,771 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:25,771 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:25,771 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:25,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38902 deadline: 1732710385768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:25,771 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:25,771 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:25,771 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:25,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38952 deadline: 1732710385768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:25,771 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:25,771 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:25,771 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:25,771 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:25,772 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:25,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38986 deadline: 1732710385769, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:25,772 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:25,772 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:25,772 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:26,081 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:26,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38944 deadline: 1732710386078, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:26,081 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:26,081 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:26,081 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:26,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38978 deadline: 1732710386078, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:26,081 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:26,081 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:26,081 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:26,081 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:26,081 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:26,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38902 deadline: 1732710386078, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:26,081 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:26,082 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:26,082 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:26,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38952 deadline: 1732710386078, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:26,082 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:26,082 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:26,082 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:26,082 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:26,082 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:26,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38986 deadline: 1732710386078, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:26,082 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:26,082 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:26,082 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:26,115 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=233 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/.tmp/C/63b7c461bfc840daa82a35801d6ddeec 2024-11-27T12:25:26,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/.tmp/A/c493b74a85404fd5ac67218aceff4053 as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/A/c493b74a85404fd5ac67218aceff4053 2024-11-27T12:25:26,123 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/A/c493b74a85404fd5ac67218aceff4053, entries=150, sequenceid=233, filesize=11.9 K 2024-11-27T12:25:26,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/.tmp/B/0da8d57788a549c7a9e737ad8269a4cc as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/B/0da8d57788a549c7a9e737ad8269a4cc 2024-11-27T12:25:26,128 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/B/0da8d57788a549c7a9e737ad8269a4cc, entries=150, sequenceid=233, filesize=11.9 K 2024-11-27T12:25:26,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/.tmp/C/63b7c461bfc840daa82a35801d6ddeec as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/C/63b7c461bfc840daa82a35801d6ddeec 2024-11-27T12:25:26,132 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/C/63b7c461bfc840daa82a35801d6ddeec, entries=150, sequenceid=233, filesize=11.9 K 2024-11-27T12:25:26,133 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=107.34 KB/109920 for 3c3055e4b5609cf4625dfa6b13256b83 in 848ms, sequenceid=233, compaction requested=true 2024-11-27T12:25:26,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2603): Flush status journal for 3c3055e4b5609cf4625dfa6b13256b83: 2024-11-27T12:25:26,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83. 2024-11-27T12:25:26,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=73 2024-11-27T12:25:26,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36215 {}] master.HMaster(4169): Remote procedure done, pid=73 2024-11-27T12:25:26,136 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=73, resume processing ppid=72 2024-11-27T12:25:26,136 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=73, ppid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0040 sec 2024-11-27T12:25:26,138 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees in 1.0080 sec 2024-11-27T12:25:26,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-27T12:25:26,268 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T12:25:26,268 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36215 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-27T12:25:26,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36215 {}] procedure2.ProcedureExecutor(1139): Stored pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees 2024-11-27T12:25:26,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-27T12:25:26,270 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T12:25:26,270 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T12:25:26,270 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=75, ppid=74, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T12:25:26,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-27T12:25:26,422 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-27T12:25:26,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83. 2024-11-27T12:25:26,423 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2902): Flushing 3c3055e4b5609cf4625dfa6b13256b83 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-11-27T12:25:26,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/.tmp/A/66312ea86dbb4acc8ef6006c989a0a61 is 50, key is test_row_0/A:col10/1732710325445/Put/seqid=0 2024-11-27T12:25:26,430 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742105_1281 (size=12151) 2024-11-27T12:25:26,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-27T12:25:26,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(8855): Flush requested on 3c3055e4b5609cf4625dfa6b13256b83 2024-11-27T12:25:26,591 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83. as already flushing 2024-11-27T12:25:26,608 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:26,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38944 deadline: 1732710386604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:26,609 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:26,609 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:26,609 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:26,612 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:26,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38978 deadline: 1732710386607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:26,612 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:26,612 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:26,612 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:26,613 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:26,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38902 deadline: 1732710386608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:26,614 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:26,614 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:26,614 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:26,614 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:26,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38986 deadline: 1732710386608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:26,614 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:26,614 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:26,614 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:26,615 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:26,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38952 deadline: 1732710386609, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:26,615 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:26,615 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:26,615 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:26,721 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:26,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38944 deadline: 1732710386717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:26,721 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:26,721 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:26,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38978 deadline: 1732710386718, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:26,721 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:26,721 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:26,721 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:26,721 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:26,721 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:26,721 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:26,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38902 deadline: 1732710386718, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:26,722 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:26,722 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:26,722 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:26,722 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:26,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38986 deadline: 1732710386718, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:26,722 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:26,722 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:26,722 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:26,722 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:26,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38952 deadline: 1732710386718, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:26,722 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:26,722 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:26,722 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:26,831 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=252 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/.tmp/A/66312ea86dbb4acc8ef6006c989a0a61 2024-11-27T12:25:26,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/.tmp/B/1bb068ecb3104d7284298caeccde01d9 is 50, key is test_row_0/B:col10/1732710325445/Put/seqid=0 2024-11-27T12:25:26,845 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742106_1282 (size=12151) 2024-11-27T12:25:26,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-27T12:25:26,930 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:26,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38944 deadline: 1732710386928, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:26,930 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:26,930 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:26,930 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:26,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38978 deadline: 1732710386928, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:26,930 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:26,931 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:26,931 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:26,931 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:26,931 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:26,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38902 deadline: 1732710386928, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:26,931 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:26,931 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:26,931 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:26,931 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:26,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38986 deadline: 1732710386928, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:26,932 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:26,932 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:26,932 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:26,932 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:26,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38952 deadline: 1732710386928, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:26,932 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:26,932 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:26,932 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:27,241 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:27,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38944 deadline: 1732710387238, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:27,241 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:27,241 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:27,241 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:27,241 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:27,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38978 deadline: 1732710387238, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:27,241 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:27,241 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:27,241 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:27,241 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:27,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38902 deadline: 1732710387238, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:27,242 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:27,242 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:27,242 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:27,242 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:27,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38986 deadline: 1732710387238, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:27,242 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:27,242 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:27,242 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:27,242 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:27,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38952 deadline: 1732710387238, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:27,242 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:27,242 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:27,242 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:27,246 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=252 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/.tmp/B/1bb068ecb3104d7284298caeccde01d9 2024-11-27T12:25:27,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/.tmp/C/f4fe3f0978584b7da5fbf1fbb01c6cf4 is 50, key is test_row_0/C:col10/1732710325445/Put/seqid=0 2024-11-27T12:25:27,256 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742107_1283 (size=12151) 2024-11-27T12:25:27,257 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=252 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/.tmp/C/f4fe3f0978584b7da5fbf1fbb01c6cf4 2024-11-27T12:25:27,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/.tmp/A/66312ea86dbb4acc8ef6006c989a0a61 as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/A/66312ea86dbb4acc8ef6006c989a0a61 2024-11-27T12:25:27,264 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/A/66312ea86dbb4acc8ef6006c989a0a61, entries=150, sequenceid=252, filesize=11.9 K 2024-11-27T12:25:27,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/.tmp/B/1bb068ecb3104d7284298caeccde01d9 as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/B/1bb068ecb3104d7284298caeccde01d9 2024-11-27T12:25:27,268 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/B/1bb068ecb3104d7284298caeccde01d9, entries=150, sequenceid=252, filesize=11.9 K 2024-11-27T12:25:27,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/.tmp/C/f4fe3f0978584b7da5fbf1fbb01c6cf4 as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/C/f4fe3f0978584b7da5fbf1fbb01c6cf4 2024-11-27T12:25:27,273 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/C/f4fe3f0978584b7da5fbf1fbb01c6cf4, entries=150, sequenceid=252, filesize=11.9 K 2024-11-27T12:25:27,274 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=100.63 KB/103050 for 3c3055e4b5609cf4625dfa6b13256b83 in 851ms, sequenceid=252, compaction requested=true 2024-11-27T12:25:27,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2603): Flush status journal for 3c3055e4b5609cf4625dfa6b13256b83: 2024-11-27T12:25:27,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83. 2024-11-27T12:25:27,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=75 2024-11-27T12:25:27,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36215 {}] master.HMaster(4169): Remote procedure done, pid=75 2024-11-27T12:25:27,278 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=75, resume processing ppid=74 2024-11-27T12:25:27,278 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=75, ppid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0060 sec 2024-11-27T12:25:27,279 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees in 1.0100 sec 2024-11-27T12:25:27,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-27T12:25:27,409 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T12:25:27,409 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36215 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-27T12:25:27,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36215 {}] procedure2.ProcedureExecutor(1139): Stored pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees 2024-11-27T12:25:27,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-27T12:25:27,411 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T12:25:27,411 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T12:25:27,411 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=77, ppid=76, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T12:25:27,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-27T12:25:27,563 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-27T12:25:27,563 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83. 2024-11-27T12:25:27,564 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2902): Flushing 3c3055e4b5609cf4625dfa6b13256b83 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-11-27T12:25:27,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/.tmp/A/e27aa87537994655bda6a2daae66f89c is 50, key is test_row_0/A:col10/1732710326607/Put/seqid=0 2024-11-27T12:25:27,589 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742108_1284 (size=12301) 2024-11-27T12:25:27,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-27T12:25:27,751 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83. as already flushing 2024-11-27T12:25:27,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(8855): Flush requested on 3c3055e4b5609cf4625dfa6b13256b83 2024-11-27T12:25:27,796 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:27,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38986 deadline: 1732710387791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:27,796 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:27,796 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:27,796 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:27,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38902 deadline: 1732710387792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:27,796 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:27,796 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:27,796 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:27,796 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:27,797 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:27,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38978 deadline: 1732710387792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:27,797 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:27,797 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:27,797 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:27,797 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:27,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38952 deadline: 1732710387792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:27,797 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:27,797 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:27,797 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:27,797 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:27,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38944 deadline: 1732710387793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:27,798 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:27,798 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:27,798 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:27,899 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:27,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38986 deadline: 1732710387898, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:27,900 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:27,900 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:27,900 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:27,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38902 deadline: 1732710387898, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:27,900 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:27,900 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:27,900 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:27,900 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:27,911 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:27,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38978 deadline: 1732710387908, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:27,911 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:27,911 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:27,911 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:27,911 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:27,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38952 deadline: 1732710387908, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:27,911 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:27,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38944 deadline: 1732710387908, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:27,912 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:27,912 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:27,912 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:27,912 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:27,912 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:27,912 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:27,990 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=270 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/.tmp/A/e27aa87537994655bda6a2daae66f89c 2024-11-27T12:25:27,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/.tmp/B/d5e23f3f4c8e4c8dbf926fe3d3a74c1a is 50, key is test_row_0/B:col10/1732710326607/Put/seqid=0 2024-11-27T12:25:28,011 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742109_1285 (size=12301) 2024-11-27T12:25:28,011 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=270 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/.tmp/B/d5e23f3f4c8e4c8dbf926fe3d3a74c1a 2024-11-27T12:25:28,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/.tmp/C/85b8bd316f1b459ba34fe66f17a05c9a is 50, key is test_row_0/C:col10/1732710326607/Put/seqid=0 2024-11-27T12:25:28,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-27T12:25:28,042 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742110_1286 (size=12301) 2024-11-27T12:25:28,043 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=270 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/.tmp/C/85b8bd316f1b459ba34fe66f17a05c9a 2024-11-27T12:25:28,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/.tmp/A/e27aa87537994655bda6a2daae66f89c as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/A/e27aa87537994655bda6a2daae66f89c 2024-11-27T12:25:28,056 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/A/e27aa87537994655bda6a2daae66f89c, entries=150, sequenceid=270, filesize=12.0 K 2024-11-27T12:25:28,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/.tmp/B/d5e23f3f4c8e4c8dbf926fe3d3a74c1a as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/B/d5e23f3f4c8e4c8dbf926fe3d3a74c1a 2024-11-27T12:25:28,063 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/B/d5e23f3f4c8e4c8dbf926fe3d3a74c1a, entries=150, sequenceid=270, filesize=12.0 K 2024-11-27T12:25:28,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/.tmp/C/85b8bd316f1b459ba34fe66f17a05c9a as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/C/85b8bd316f1b459ba34fe66f17a05c9a 2024-11-27T12:25:28,069 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/C/85b8bd316f1b459ba34fe66f17a05c9a, entries=150, sequenceid=270, filesize=12.0 K 2024-11-27T12:25:28,070 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=107.34 KB/109920 for 3c3055e4b5609cf4625dfa6b13256b83 in 507ms, sequenceid=270, compaction requested=true 2024-11-27T12:25:28,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2603): Flush status journal for 3c3055e4b5609cf4625dfa6b13256b83: 2024-11-27T12:25:28,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83. 2024-11-27T12:25:28,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=77 2024-11-27T12:25:28,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36215 {}] master.HMaster(4169): Remote procedure done, pid=77 2024-11-27T12:25:28,074 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=77, resume processing ppid=76 2024-11-27T12:25:28,074 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=77, ppid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 661 msec 2024-11-27T12:25:28,075 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees in 666 msec 2024-11-27T12:25:28,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(8855): Flush requested on 3c3055e4b5609cf4625dfa6b13256b83 2024-11-27T12:25:28,113 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T12:25:28,113 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3c3055e4b5609cf4625dfa6b13256b83 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-27T12:25:28,118 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/.tmp/A/cd91570099844461bf47f14d50dad1a6 is 50, key is test_row_0/A:col10/1732710327791/Put/seqid=0 2024-11-27T12:25:28,125 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742111_1287 (size=14741) 2024-11-27T12:25:28,137 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:28,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38978 deadline: 1732710388129, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:28,137 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:28,137 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:28,137 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:28,138 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:28,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38952 deadline: 1732710388132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:28,139 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:28,139 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:28,139 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:28,139 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:28,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38944 deadline: 1732710388133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:28,139 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:28,139 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:28,139 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:28,144 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:28,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38986 deadline: 1732710388137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:28,145 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:28,145 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:28,145 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:28,145 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:28,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38902 deadline: 1732710388137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:28,145 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:28,145 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:28,145 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:28,251 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:28,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38978 deadline: 1732710388248, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:28,252 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:28,252 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:28,252 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:28,252 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:28,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38952 deadline: 1732710388248, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:28,252 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:28,252 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:28,252 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:28,252 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:28,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38944 deadline: 1732710388248, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:28,253 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:28,253 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:28,253 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:28,253 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:28,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38902 deadline: 1732710388248, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:28,253 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:28,253 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:28,253 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:28,253 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:28,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38986 deadline: 1732710388248, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:28,253 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:28,253 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:28,253 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:28,461 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:28,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38978 deadline: 1732710388458, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:28,462 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:28,462 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:28,462 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:28,462 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:28,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38952 deadline: 1732710388458, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:28,462 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:28,462 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:28,462 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:28,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38944 deadline: 1732710388458, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:28,462 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:28,462 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:28,462 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:28,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38902 deadline: 1732710388458, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:28,462 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:28,462 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:28,462 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:28,462 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:28,462 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:28,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38986 deadline: 1732710388458, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:28,462 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:28,463 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:28,463 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:28,463 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:28,526 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=290 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/.tmp/A/cd91570099844461bf47f14d50dad1a6 2024-11-27T12:25:28,533 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/.tmp/B/3f25b0c093f8408dbe2df41b93c368f4 is 50, key is test_row_0/B:col10/1732710327791/Put/seqid=0 2024-11-27T12:25:28,536 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742112_1288 (size=12301) 2024-11-27T12:25:28,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-27T12:25:28,549 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T12:25:28,549 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36215 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-27T12:25:28,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36215 {}] procedure2.ProcedureExecutor(1139): Stored pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees 2024-11-27T12:25:28,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-27T12:25:28,551 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T12:25:28,551 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T12:25:28,551 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=79, ppid=78, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T12:25:28,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-27T12:25:28,704 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-27T12:25:28,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83. 2024-11-27T12:25:28,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83. as already flushing 2024-11-27T12:25:28,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83. 2024-11-27T12:25:28,704 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => 3c3055e4b5609cf4625dfa6b13256b83, NAME => 'TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:25:28,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => 3c3055e4b5609cf4625dfa6b13256b83, NAME => 'TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:25:28,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36215 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3c3055e4b5609cf4625dfa6b13256b83, NAME => 'TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3c3055e4b5609cf4625dfa6b13256b83, NAME => 'TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:25:28,771 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:28,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38978 deadline: 1732710388768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:28,771 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:28,771 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:28,771 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:28,771 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:28,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38952 deadline: 1732710388768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:28,771 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:28,771 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:28,771 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:28,772 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:28,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38944 deadline: 1732710388768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:28,772 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:28,772 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:28,772 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:28,772 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:28,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38902 deadline: 1732710388768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:28,773 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:28,773 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:28,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38986 deadline: 1732710388768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:28,773 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:28,773 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:28,773 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:28,773 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:28,773 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:28,857 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-27T12:25:28,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83. 2024-11-27T12:25:28,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83. as already flushing 2024-11-27T12:25:28,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83. 2024-11-27T12:25:28,857 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => 3c3055e4b5609cf4625dfa6b13256b83, NAME => 'TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:25:28,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => 3c3055e4b5609cf4625dfa6b13256b83, NAME => 'TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:25:28,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36215 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3c3055e4b5609cf4625dfa6b13256b83, NAME => 'TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3c3055e4b5609cf4625dfa6b13256b83, NAME => 'TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:25:28,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-27T12:25:28,937 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=290 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/.tmp/B/3f25b0c093f8408dbe2df41b93c368f4 2024-11-27T12:25:28,943 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/.tmp/C/21e3ef7a25ef4826b19cd77d44bf6736 is 50, key is test_row_0/C:col10/1732710327791/Put/seqid=0 2024-11-27T12:25:28,946 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742113_1289 (size=12301) 2024-11-27T12:25:29,009 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-27T12:25:29,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83. 2024-11-27T12:25:29,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83. as already flushing 2024-11-27T12:25:29,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83. 2024-11-27T12:25:29,010 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => 3c3055e4b5609cf4625dfa6b13256b83, NAME => 'TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:25:29,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => 3c3055e4b5609cf4625dfa6b13256b83, NAME => 'TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:25:29,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36215 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3c3055e4b5609cf4625dfa6b13256b83, NAME => 'TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3c3055e4b5609cf4625dfa6b13256b83, NAME => 'TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:25:29,162 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-27T12:25:29,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83. 2024-11-27T12:25:29,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83. as already flushing 2024-11-27T12:25:29,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83. 2024-11-27T12:25:29,163 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => 3c3055e4b5609cf4625dfa6b13256b83, NAME => 'TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:25:29,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => 3c3055e4b5609cf4625dfa6b13256b83, NAME => 'TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:25:29,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36215 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3c3055e4b5609cf4625dfa6b13256b83, NAME => 'TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3c3055e4b5609cf4625dfa6b13256b83, NAME => 'TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:25:29,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-27T12:25:29,280 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:29,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38978 deadline: 1732710389277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:29,280 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:29,280 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:29,280 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:29,281 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:29,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38952 deadline: 1732710389278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:29,281 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:29,281 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:29,282 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:29,282 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:29,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38944 deadline: 1732710389278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:29,282 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:29,282 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:29,282 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:29,282 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:29,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38902 deadline: 1732710389278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:29,282 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:29,282 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:29,282 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:29,282 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:29,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38986 deadline: 1732710389278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:29,282 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:29,283 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:29,283 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:29,315 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-27T12:25:29,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83. 2024-11-27T12:25:29,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83. as already flushing 2024-11-27T12:25:29,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83. 2024-11-27T12:25:29,316 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => 3c3055e4b5609cf4625dfa6b13256b83, NAME => 'TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:25:29,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => 3c3055e4b5609cf4625dfa6b13256b83, NAME => 'TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:25:29,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36215 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3c3055e4b5609cf4625dfa6b13256b83, NAME => 'TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3c3055e4b5609cf4625dfa6b13256b83, NAME => 'TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:25:29,347 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=290 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/.tmp/C/21e3ef7a25ef4826b19cd77d44bf6736 2024-11-27T12:25:29,351 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/.tmp/A/cd91570099844461bf47f14d50dad1a6 as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/A/cd91570099844461bf47f14d50dad1a6 2024-11-27T12:25:29,354 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/A/cd91570099844461bf47f14d50dad1a6, entries=200, sequenceid=290, filesize=14.4 K 2024-11-27T12:25:29,355 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/.tmp/B/3f25b0c093f8408dbe2df41b93c368f4 as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/B/3f25b0c093f8408dbe2df41b93c368f4 2024-11-27T12:25:29,358 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/B/3f25b0c093f8408dbe2df41b93c368f4, entries=150, sequenceid=290, filesize=12.0 K 2024-11-27T12:25:29,359 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/.tmp/C/21e3ef7a25ef4826b19cd77d44bf6736 as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/C/21e3ef7a25ef4826b19cd77d44bf6736 2024-11-27T12:25:29,363 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/C/21e3ef7a25ef4826b19cd77d44bf6736, entries=150, sequenceid=290, filesize=12.0 K 2024-11-27T12:25:29,363 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=87.22 KB/89310 for 3c3055e4b5609cf4625dfa6b13256b83 in 1250ms, sequenceid=290, compaction requested=true 2024-11-27T12:25:29,364 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3c3055e4b5609cf4625dfa6b13256b83: 2024-11-27T12:25:29,364 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3c3055e4b5609cf4625dfa6b13256b83:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T12:25:29,364 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T12:25:29,364 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 7 store files, 0 compacting, 7 eligible, 16 blocking 2024-11-27T12:25:29,364 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3c3055e4b5609cf4625dfa6b13256b83:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T12:25:29,364 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T12:25:29,364 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 7 store files, 0 compacting, 7 eligible, 16 blocking 2024-11-27T12:25:29,364 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3c3055e4b5609cf4625dfa6b13256b83:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T12:25:29,364 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T12:25:29,365 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 7 files of size 85767 starting at candidate #0 after considering 15 permutations with 15 in ratio 2024-11-27T12:25:29,365 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 7 files of size 88207 starting at candidate #0 after considering 15 permutations with 15 in ratio 2024-11-27T12:25:29,365 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HStore(1541): 3c3055e4b5609cf4625dfa6b13256b83/B is initiating minor compaction (all files) 2024-11-27T12:25:29,365 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HStore(1541): 3c3055e4b5609cf4625dfa6b13256b83/A is initiating minor compaction (all files) 2024-11-27T12:25:29,365 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3c3055e4b5609cf4625dfa6b13256b83/A in TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83. 2024-11-27T12:25:29,365 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3c3055e4b5609cf4625dfa6b13256b83/B in TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83. 2024-11-27T12:25:29,366 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/A/0bed16d005ea457f849e554a21ba34d3, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/A/bee2be8112594615bbc186c2d5260738, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/A/5b554eb6968b4e12b99cb76384381b1b, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/A/c493b74a85404fd5ac67218aceff4053, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/A/66312ea86dbb4acc8ef6006c989a0a61, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/A/e27aa87537994655bda6a2daae66f89c, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/A/cd91570099844461bf47f14d50dad1a6] into tmpdir=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/.tmp, totalSize=86.1 K 2024-11-27T12:25:29,366 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/B/b4b29f12a9c04d66b837557a085f8c0d, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/B/e72b07f730c74167ab92afa7ab900b2e, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/B/6f72a4387d204bda9ce5046d5117891e, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/B/0da8d57788a549c7a9e737ad8269a4cc, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/B/1bb068ecb3104d7284298caeccde01d9, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/B/d5e23f3f4c8e4c8dbf926fe3d3a74c1a, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/B/3f25b0c093f8408dbe2df41b93c368f4] into tmpdir=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/.tmp, totalSize=83.8 K 2024-11-27T12:25:29,366 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.Compactor(225): Compacting b4b29f12a9c04d66b837557a085f8c0d, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=177, earliestPutTs=1732710321303 2024-11-27T12:25:29,366 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0bed16d005ea457f849e554a21ba34d3, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=177, earliestPutTs=1732710321303 2024-11-27T12:25:29,366 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.Compactor(225): Compacting e72b07f730c74167ab92afa7ab900b2e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1732710322456 2024-11-27T12:25:29,366 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] compactions.Compactor(225): Compacting bee2be8112594615bbc186c2d5260738, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1732710322456 2024-11-27T12:25:29,367 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5b554eb6968b4e12b99cb76384381b1b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=216, earliestPutTs=1732710323123 2024-11-27T12:25:29,367 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.Compactor(225): Compacting 6f72a4387d204bda9ce5046d5117891e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=216, earliestPutTs=1732710323123 2024-11-27T12:25:29,367 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.Compactor(225): Compacting 0da8d57788a549c7a9e737ad8269a4cc, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1732710324281 2024-11-27T12:25:29,367 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] compactions.Compactor(225): Compacting c493b74a85404fd5ac67218aceff4053, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1732710324281 2024-11-27T12:25:29,367 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.Compactor(225): Compacting 1bb068ecb3104d7284298caeccde01d9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=252, earliestPutTs=1732710325445 2024-11-27T12:25:29,367 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] compactions.Compactor(225): Compacting 66312ea86dbb4acc8ef6006c989a0a61, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=252, earliestPutTs=1732710325445 2024-11-27T12:25:29,368 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.Compactor(225): Compacting d5e23f3f4c8e4c8dbf926fe3d3a74c1a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=270, earliestPutTs=1732710326603 2024-11-27T12:25:29,368 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] compactions.Compactor(225): Compacting e27aa87537994655bda6a2daae66f89c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=270, earliestPutTs=1732710326603 2024-11-27T12:25:29,368 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.Compactor(225): Compacting 3f25b0c093f8408dbe2df41b93c368f4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=290, earliestPutTs=1732710327791 2024-11-27T12:25:29,368 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] compactions.Compactor(225): Compacting cd91570099844461bf47f14d50dad1a6, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=290, earliestPutTs=1732710327790 2024-11-27T12:25:29,382 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3c3055e4b5609cf4625dfa6b13256b83#B#compaction#244 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-27T12:25:29,382 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3c3055e4b5609cf4625dfa6b13256b83#A#compaction#243 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T12:25:29,382 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/.tmp/B/23641de6b8364a86b271b4cabc02c466 is 50, key is test_row_0/B:col10/1732710327791/Put/seqid=0 2024-11-27T12:25:29,382 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/.tmp/A/cd36429e71aa4e649966af3901833a50 is 50, key is test_row_0/A:col10/1732710327791/Put/seqid=0 2024-11-27T12:25:29,389 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742114_1290 (size=12949) 2024-11-27T12:25:29,390 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742115_1291 (size=12949) 2024-11-27T12:25:29,468 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-27T12:25:29,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83. 2024-11-27T12:25:29,468 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2902): Flushing 3c3055e4b5609cf4625dfa6b13256b83 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-27T12:25:29,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/.tmp/A/2b0c295e53fd41b7bc839062c8979128 is 50, key is test_row_0/A:col10/1732710328127/Put/seqid=0 2024-11-27T12:25:29,476 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742116_1292 (size=12301) 2024-11-27T12:25:29,477 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=306 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/.tmp/A/2b0c295e53fd41b7bc839062c8979128 2024-11-27T12:25:29,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/.tmp/B/1c94e93a7bfc48f39c2649990c47802c is 50, key is test_row_0/B:col10/1732710328127/Put/seqid=0 2024-11-27T12:25:29,486 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742117_1293 (size=12301) 2024-11-27T12:25:29,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-27T12:25:29,799 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/.tmp/A/cd36429e71aa4e649966af3901833a50 as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/A/cd36429e71aa4e649966af3901833a50 2024-11-27T12:25:29,799 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/.tmp/B/23641de6b8364a86b271b4cabc02c466 as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/B/23641de6b8364a86b271b4cabc02c466 2024-11-27T12:25:29,805 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 7 (all) file(s) in 3c3055e4b5609cf4625dfa6b13256b83/A of 3c3055e4b5609cf4625dfa6b13256b83 into cd36429e71aa4e649966af3901833a50(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T12:25:29,805 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 7 (all) file(s) in 3c3055e4b5609cf4625dfa6b13256b83/B of 3c3055e4b5609cf4625dfa6b13256b83 into 23641de6b8364a86b271b4cabc02c466(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T12:25:29,805 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3c3055e4b5609cf4625dfa6b13256b83: 2024-11-27T12:25:29,805 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3c3055e4b5609cf4625dfa6b13256b83: 2024-11-27T12:25:29,805 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., storeName=3c3055e4b5609cf4625dfa6b13256b83/A, priority=9, startTime=1732710329364; duration=0sec 2024-11-27T12:25:29,805 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., storeName=3c3055e4b5609cf4625dfa6b13256b83/B, priority=9, startTime=1732710329364; duration=0sec 2024-11-27T12:25:29,805 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T12:25:29,805 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3c3055e4b5609cf4625dfa6b13256b83:A 2024-11-27T12:25:29,805 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T12:25:29,805 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 7 store files, 0 compacting, 7 eligible, 16 blocking 2024-11-27T12:25:29,805 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3c3055e4b5609cf4625dfa6b13256b83:B 2024-11-27T12:25:29,807 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 7 files of size 85767 starting at candidate #0 after considering 15 permutations with 15 in ratio 2024-11-27T12:25:29,807 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HStore(1541): 3c3055e4b5609cf4625dfa6b13256b83/C is initiating minor compaction (all files) 2024-11-27T12:25:29,807 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3c3055e4b5609cf4625dfa6b13256b83/C in TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83. 2024-11-27T12:25:29,807 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/C/fdc5ebc404ba4ff39280b4e5ef60cf65, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/C/96775b63a8f94aeabe40c656fdd1b0fa, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/C/328a92f8c4d9445ab7c91fbfcc76c670, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/C/63b7c461bfc840daa82a35801d6ddeec, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/C/f4fe3f0978584b7da5fbf1fbb01c6cf4, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/C/85b8bd316f1b459ba34fe66f17a05c9a, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/C/21e3ef7a25ef4826b19cd77d44bf6736] into tmpdir=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/.tmp, totalSize=83.8 K 2024-11-27T12:25:29,807 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] compactions.Compactor(225): Compacting fdc5ebc404ba4ff39280b4e5ef60cf65, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=177, earliestPutTs=1732710321303 2024-11-27T12:25:29,808 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] compactions.Compactor(225): Compacting 96775b63a8f94aeabe40c656fdd1b0fa, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1732710322456 2024-11-27T12:25:29,808 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] compactions.Compactor(225): Compacting 328a92f8c4d9445ab7c91fbfcc76c670, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=216, earliestPutTs=1732710323123 2024-11-27T12:25:29,808 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] compactions.Compactor(225): Compacting 63b7c461bfc840daa82a35801d6ddeec, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1732710324281 2024-11-27T12:25:29,808 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] compactions.Compactor(225): Compacting f4fe3f0978584b7da5fbf1fbb01c6cf4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=252, earliestPutTs=1732710325445 2024-11-27T12:25:29,809 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] compactions.Compactor(225): Compacting 85b8bd316f1b459ba34fe66f17a05c9a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=270, earliestPutTs=1732710326603 2024-11-27T12:25:29,809 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] compactions.Compactor(225): Compacting 21e3ef7a25ef4826b19cd77d44bf6736, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=290, earliestPutTs=1732710327791 2024-11-27T12:25:29,823 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3c3055e4b5609cf4625dfa6b13256b83#C#compaction#247 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T12:25:29,824 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/.tmp/C/8c565b953e4d4760b0e76b3c661fc4c8 is 50, key is test_row_0/C:col10/1732710327791/Put/seqid=0 2024-11-27T12:25:29,828 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742118_1294 (size=12949) 2024-11-27T12:25:29,886 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=306 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/.tmp/B/1c94e93a7bfc48f39c2649990c47802c 2024-11-27T12:25:29,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/.tmp/C/82f3864ec8f342e7be9d13ceb7ebb340 is 50, key is test_row_0/C:col10/1732710328127/Put/seqid=0 2024-11-27T12:25:29,898 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742119_1295 (size=12301) 2024-11-27T12:25:29,899 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=306 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/.tmp/C/82f3864ec8f342e7be9d13ceb7ebb340 2024-11-27T12:25:29,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/.tmp/A/2b0c295e53fd41b7bc839062c8979128 as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/A/2b0c295e53fd41b7bc839062c8979128 2024-11-27T12:25:29,913 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/A/2b0c295e53fd41b7bc839062c8979128, entries=150, sequenceid=306, filesize=12.0 K 2024-11-27T12:25:29,914 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/.tmp/B/1c94e93a7bfc48f39c2649990c47802c as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/B/1c94e93a7bfc48f39c2649990c47802c 2024-11-27T12:25:29,920 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/B/1c94e93a7bfc48f39c2649990c47802c, entries=150, sequenceid=306, filesize=12.0 K 2024-11-27T12:25:29,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/.tmp/C/82f3864ec8f342e7be9d13ceb7ebb340 as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/C/82f3864ec8f342e7be9d13ceb7ebb340 2024-11-27T12:25:29,925 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/C/82f3864ec8f342e7be9d13ceb7ebb340, entries=150, sequenceid=306, filesize=12.0 K 2024-11-27T12:25:29,925 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=0 B/0 for 3c3055e4b5609cf4625dfa6b13256b83 in 457ms, sequenceid=306, compaction requested=false 2024-11-27T12:25:29,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2603): Flush status journal for 3c3055e4b5609cf4625dfa6b13256b83: 2024-11-27T12:25:29,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83. 2024-11-27T12:25:29,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=79 2024-11-27T12:25:29,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36215 {}] master.HMaster(4169): Remote procedure done, pid=79 2024-11-27T12:25:29,932 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=79, resume processing ppid=78 2024-11-27T12:25:29,932 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=79, ppid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3760 sec 2024-11-27T12:25:29,935 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees in 1.3840 sec 2024-11-27T12:25:30,014 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-27T12:25:30,232 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/.tmp/C/8c565b953e4d4760b0e76b3c661fc4c8 as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/C/8c565b953e4d4760b0e76b3c661fc4c8 2024-11-27T12:25:30,244 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 7 (all) file(s) in 3c3055e4b5609cf4625dfa6b13256b83/C of 3c3055e4b5609cf4625dfa6b13256b83 into 8c565b953e4d4760b0e76b3c661fc4c8(size=12.6 K), total size for store is 24.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T12:25:30,244 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3c3055e4b5609cf4625dfa6b13256b83: 2024-11-27T12:25:30,244 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., storeName=3c3055e4b5609cf4625dfa6b13256b83/C, priority=9, startTime=1732710329364; duration=0sec 2024-11-27T12:25:30,244 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T12:25:30,244 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3c3055e4b5609cf4625dfa6b13256b83:C 2024-11-27T12:25:30,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(8855): Flush requested on 3c3055e4b5609cf4625dfa6b13256b83 2024-11-27T12:25:30,329 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T12:25:30,329 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3c3055e4b5609cf4625dfa6b13256b83 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-27T12:25:30,333 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/.tmp/A/597c37a28dc54656a5185dedeb61b32f is 50, key is test_row_0/A:col10/1732710330326/Put/seqid=0 2024-11-27T12:25:30,341 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742120_1296 (size=12301) 2024-11-27T12:25:30,356 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:30,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38978 deadline: 1732710390351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:30,356 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:30,356 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:30,356 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:30,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38944 deadline: 1732710390352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:30,356 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:30,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38902 deadline: 1732710390352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:30,356 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:30,357 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:30,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38952 deadline: 1732710390353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:30,357 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:30,357 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:30,357 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:30,357 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:30,357 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:30,357 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:30,357 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:30,357 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:30,358 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:30,360 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:30,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38986 deadline: 1732710390356, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:30,360 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:30,360 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:30,360 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:30,461 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:30,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38978 deadline: 1732710390458, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:30,462 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:30,462 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:30,462 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:30,469 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:30,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38952 deadline: 1732710390468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:30,469 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:30,469 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:30,469 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:30,469 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:30,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38944 deadline: 1732710390468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:30,469 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:30,470 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:30,470 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:30,471 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:30,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38902 deadline: 1732710390469, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:30,471 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:30,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38986 deadline: 1732710390469, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:30,472 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:30,472 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:30,472 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:30,472 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:30,472 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:30,472 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:30,670 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:30,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38978 deadline: 1732710390668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:30,670 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:30,670 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:30,671 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:30,680 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:30,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38952 deadline: 1732710390678, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:30,681 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:30,681 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:30,681 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:30,682 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:30,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38944 deadline: 1732710390678, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:30,682 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:30,682 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:30,682 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:30,682 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:30,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38986 deadline: 1732710390679, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:30,683 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:30,683 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:30,683 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:30,683 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:30,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38902 deadline: 1732710390679, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:30,683 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:30,683 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:30,683 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:30,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-27T12:25:30,698 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T12:25:30,699 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36215 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-27T12:25:30,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36215 {}] procedure2.ProcedureExecutor(1139): Stored pid=80, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees 2024-11-27T12:25:30,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-27T12:25:30,700 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=80, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T12:25:30,701 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=80, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T12:25:30,701 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=81, ppid=80, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T12:25:30,742 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=320 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/.tmp/A/597c37a28dc54656a5185dedeb61b32f 2024-11-27T12:25:30,754 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/.tmp/B/a275d028be1f4b62817936d556f6df3c is 50, key is test_row_0/B:col10/1732710330326/Put/seqid=0 2024-11-27T12:25:30,771 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742121_1297 (size=12301) 2024-11-27T12:25:30,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-27T12:25:30,857 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-27T12:25:30,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83. 2024-11-27T12:25:30,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83. as already flushing 2024-11-27T12:25:30,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83. 2024-11-27T12:25:30,858 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 3c3055e4b5609cf4625dfa6b13256b83, NAME => 'TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:25:30,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 3c3055e4b5609cf4625dfa6b13256b83, NAME => 'TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:25:30,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36215 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3c3055e4b5609cf4625dfa6b13256b83, NAME => 'TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3c3055e4b5609cf4625dfa6b13256b83, NAME => 'TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:25:30,981 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:30,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38978 deadline: 1732710390978, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:30,981 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:30,981 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:30,981 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:30,990 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:30,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38952 deadline: 1732710390988, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:30,990 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:30,990 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:30,990 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:30,990 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:30,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38944 deadline: 1732710390988, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:30,991 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:30,991 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:30,991 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:30,991 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:30,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38986 deadline: 1732710390989, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:30,991 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:30,991 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:30,991 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:30,992 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:30,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38902 deadline: 1732710390989, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:30,992 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:30,992 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:30,992 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:31,011 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-27T12:25:31,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83. 2024-11-27T12:25:31,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83. as already flushing 2024-11-27T12:25:31,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83. 2024-11-27T12:25:31,011 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 3c3055e4b5609cf4625dfa6b13256b83, NAME => 'TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:25:31,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 3c3055e4b5609cf4625dfa6b13256b83, NAME => 'TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:25:31,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36215 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3c3055e4b5609cf4625dfa6b13256b83, NAME => 'TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3c3055e4b5609cf4625dfa6b13256b83, NAME => 'TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:25:31,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-27T12:25:31,164 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-27T12:25:31,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83. 2024-11-27T12:25:31,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83. as already flushing 2024-11-27T12:25:31,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83. 2024-11-27T12:25:31,165 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 3c3055e4b5609cf4625dfa6b13256b83, NAME => 'TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:25:31,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 3c3055e4b5609cf4625dfa6b13256b83, NAME => 'TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:25:31,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36215 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3c3055e4b5609cf4625dfa6b13256b83, NAME => 'TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3c3055e4b5609cf4625dfa6b13256b83, NAME => 'TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:25:31,172 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=320 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/.tmp/B/a275d028be1f4b62817936d556f6df3c 2024-11-27T12:25:31,177 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/.tmp/C/0430b886bcd54740b8e10bc9b48f7997 is 50, key is test_row_0/C:col10/1732710330326/Put/seqid=0 2024-11-27T12:25:31,181 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742122_1298 (size=12301) 2024-11-27T12:25:31,317 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-27T12:25:31,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83. 2024-11-27T12:25:31,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83. as already flushing 2024-11-27T12:25:31,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83. 2024-11-27T12:25:31,317 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 3c3055e4b5609cf4625dfa6b13256b83, NAME => 'TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:25:31,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 3c3055e4b5609cf4625dfa6b13256b83, NAME => 'TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:25:31,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36215 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3c3055e4b5609cf4625dfa6b13256b83, NAME => 'TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3c3055e4b5609cf4625dfa6b13256b83, NAME => 'TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:25:31,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-27T12:25:31,470 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-27T12:25:31,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83. 2024-11-27T12:25:31,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83. as already flushing 2024-11-27T12:25:31,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83. 2024-11-27T12:25:31,470 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 3c3055e4b5609cf4625dfa6b13256b83, NAME => 'TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:25:31,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 3c3055e4b5609cf4625dfa6b13256b83, NAME => 'TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:25:31,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36215 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3c3055e4b5609cf4625dfa6b13256b83, NAME => 'TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3c3055e4b5609cf4625dfa6b13256b83, NAME => 'TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:25:31,491 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:31,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38978 deadline: 1732710391488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:31,491 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:31,491 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:31,491 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:31,500 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:31,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38952 deadline: 1732710391498, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:31,500 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:31,500 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:31,500 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:31,500 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:31,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38944 deadline: 1732710391498, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:31,501 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:31,501 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:31,501 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:31,501 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:31,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38986 deadline: 1732710391498, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:31,501 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:31,501 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:31,501 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:31,501 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:31,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38902 deadline: 1732710391498, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:31,501 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:31,501 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:31,501 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:31,581 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=320 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/.tmp/C/0430b886bcd54740b8e10bc9b48f7997 2024-11-27T12:25:31,586 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/.tmp/A/597c37a28dc54656a5185dedeb61b32f as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/A/597c37a28dc54656a5185dedeb61b32f 2024-11-27T12:25:31,592 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/A/597c37a28dc54656a5185dedeb61b32f, entries=150, sequenceid=320, filesize=12.0 K 2024-11-27T12:25:31,594 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/.tmp/B/a275d028be1f4b62817936d556f6df3c as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/B/a275d028be1f4b62817936d556f6df3c 2024-11-27T12:25:31,599 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/B/a275d028be1f4b62817936d556f6df3c, entries=150, sequenceid=320, filesize=12.0 K 2024-11-27T12:25:31,600 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/.tmp/C/0430b886bcd54740b8e10bc9b48f7997 as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/C/0430b886bcd54740b8e10bc9b48f7997 2024-11-27T12:25:31,604 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/C/0430b886bcd54740b8e10bc9b48f7997, entries=150, sequenceid=320, filesize=12.0 K 2024-11-27T12:25:31,605 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 3c3055e4b5609cf4625dfa6b13256b83 in 1276ms, sequenceid=320, compaction requested=true 2024-11-27T12:25:31,605 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3c3055e4b5609cf4625dfa6b13256b83: 2024-11-27T12:25:31,605 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3c3055e4b5609cf4625dfa6b13256b83:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T12:25:31,605 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T12:25:31,605 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T12:25:31,605 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3c3055e4b5609cf4625dfa6b13256b83:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T12:25:31,605 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T12:25:31,605 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T12:25:31,605 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3c3055e4b5609cf4625dfa6b13256b83:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T12:25:31,605 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T12:25:31,606 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T12:25:31,606 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T12:25:31,606 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HStore(1541): 3c3055e4b5609cf4625dfa6b13256b83/A is initiating minor compaction (all files) 2024-11-27T12:25:31,606 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HStore(1541): 3c3055e4b5609cf4625dfa6b13256b83/B is initiating minor compaction (all files) 2024-11-27T12:25:31,606 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3c3055e4b5609cf4625dfa6b13256b83/A in TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83. 2024-11-27T12:25:31,606 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3c3055e4b5609cf4625dfa6b13256b83/B in TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83. 2024-11-27T12:25:31,607 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/A/cd36429e71aa4e649966af3901833a50, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/A/2b0c295e53fd41b7bc839062c8979128, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/A/597c37a28dc54656a5185dedeb61b32f] into tmpdir=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/.tmp, totalSize=36.7 K 2024-11-27T12:25:31,607 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/B/23641de6b8364a86b271b4cabc02c466, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/B/1c94e93a7bfc48f39c2649990c47802c, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/B/a275d028be1f4b62817936d556f6df3c] into tmpdir=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/.tmp, totalSize=36.7 K 2024-11-27T12:25:31,607 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.Compactor(225): Compacting 23641de6b8364a86b271b4cabc02c466, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=290, earliestPutTs=1732710327791 2024-11-27T12:25:31,607 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] compactions.Compactor(225): Compacting cd36429e71aa4e649966af3901833a50, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=290, earliestPutTs=1732710327791 2024-11-27T12:25:31,607 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.Compactor(225): Compacting 1c94e93a7bfc48f39c2649990c47802c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=306, earliestPutTs=1732710328127 2024-11-27T12:25:31,607 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2b0c295e53fd41b7bc839062c8979128, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=306, earliestPutTs=1732710328127 2024-11-27T12:25:31,607 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.Compactor(225): Compacting a275d028be1f4b62817936d556f6df3c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=320, earliestPutTs=1732710330326 2024-11-27T12:25:31,608 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] compactions.Compactor(225): Compacting 597c37a28dc54656a5185dedeb61b32f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=320, earliestPutTs=1732710330326 2024-11-27T12:25:31,622 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-27T12:25:31,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83. 2024-11-27T12:25:31,623 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2902): Flushing 3c3055e4b5609cf4625dfa6b13256b83 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-27T12:25:31,630 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3c3055e4b5609cf4625dfa6b13256b83#B#compaction#252 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T12:25:31,630 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/.tmp/B/060bec5dc8e5470e9040ef4c0f0945a8 is 50, key is test_row_0/B:col10/1732710330326/Put/seqid=0 2024-11-27T12:25:31,637 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3c3055e4b5609cf4625dfa6b13256b83#A#compaction#253 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T12:25:31,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/.tmp/A/50083cd8a6c3484284932c1edcc7fe51 is 50, key is test_row_0/A:col10/1732710330352/Put/seqid=0 2024-11-27T12:25:31,637 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/.tmp/A/fa238a787cd7411e81684fcf3a369372 is 50, key is test_row_0/A:col10/1732710330326/Put/seqid=0 2024-11-27T12:25:31,653 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742123_1299 (size=13051) 2024-11-27T12:25:31,658 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/.tmp/B/060bec5dc8e5470e9040ef4c0f0945a8 as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/B/060bec5dc8e5470e9040ef4c0f0945a8 2024-11-27T12:25:31,662 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3c3055e4b5609cf4625dfa6b13256b83/B of 3c3055e4b5609cf4625dfa6b13256b83 into 060bec5dc8e5470e9040ef4c0f0945a8(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T12:25:31,662 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3c3055e4b5609cf4625dfa6b13256b83: 2024-11-27T12:25:31,662 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., storeName=3c3055e4b5609cf4625dfa6b13256b83/B, priority=13, startTime=1732710331605; duration=0sec 2024-11-27T12:25:31,662 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T12:25:31,662 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3c3055e4b5609cf4625dfa6b13256b83:B 2024-11-27T12:25:31,662 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T12:25:31,663 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T12:25:31,663 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HStore(1541): 3c3055e4b5609cf4625dfa6b13256b83/C is initiating minor compaction (all files) 2024-11-27T12:25:31,663 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3c3055e4b5609cf4625dfa6b13256b83/C in TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83. 2024-11-27T12:25:31,663 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/C/8c565b953e4d4760b0e76b3c661fc4c8, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/C/82f3864ec8f342e7be9d13ceb7ebb340, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/C/0430b886bcd54740b8e10bc9b48f7997] into tmpdir=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/.tmp, totalSize=36.7 K 2024-11-27T12:25:31,664 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.Compactor(225): Compacting 8c565b953e4d4760b0e76b3c661fc4c8, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=290, earliestPutTs=1732710327791 2024-11-27T12:25:31,664 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.Compactor(225): Compacting 82f3864ec8f342e7be9d13ceb7ebb340, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=306, earliestPutTs=1732710328127 2024-11-27T12:25:31,665 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.Compactor(225): Compacting 0430b886bcd54740b8e10bc9b48f7997, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=320, earliestPutTs=1732710330326 2024-11-27T12:25:31,672 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742124_1300 (size=12301) 2024-11-27T12:25:31,674 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3c3055e4b5609cf4625dfa6b13256b83#C#compaction#255 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T12:25:31,675 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/.tmp/C/e914b2695c21494c8b45eb11975f936f is 50, key is test_row_0/C:col10/1732710330326/Put/seqid=0 2024-11-27T12:25:31,677 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742125_1301 (size=13051) 2024-11-27T12:25:31,678 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=345 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/.tmp/A/50083cd8a6c3484284932c1edcc7fe51 2024-11-27T12:25:31,683 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/.tmp/A/fa238a787cd7411e81684fcf3a369372 as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/A/fa238a787cd7411e81684fcf3a369372 2024-11-27T12:25:31,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/.tmp/B/1d2ae34d54bf4236912e6107e1c4872b is 50, key is test_row_0/B:col10/1732710330352/Put/seqid=0 2024-11-27T12:25:31,687 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3c3055e4b5609cf4625dfa6b13256b83/A of 3c3055e4b5609cf4625dfa6b13256b83 into fa238a787cd7411e81684fcf3a369372(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T12:25:31,688 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3c3055e4b5609cf4625dfa6b13256b83: 2024-11-27T12:25:31,688 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., storeName=3c3055e4b5609cf4625dfa6b13256b83/A, priority=13, startTime=1732710331605; duration=0sec 2024-11-27T12:25:31,688 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T12:25:31,688 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3c3055e4b5609cf4625dfa6b13256b83:A 2024-11-27T12:25:31,691 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742127_1303 (size=12301) 2024-11-27T12:25:31,705 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742126_1302 (size=13051) 2024-11-27T12:25:31,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-27T12:25:32,093 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=345 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/.tmp/B/1d2ae34d54bf4236912e6107e1c4872b 2024-11-27T12:25:32,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/.tmp/C/9bf6b334ff694e30b39c574c6ad8d938 is 50, key is test_row_0/C:col10/1732710330352/Put/seqid=0 2024-11-27T12:25:32,105 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/.tmp/C/e914b2695c21494c8b45eb11975f936f as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/C/e914b2695c21494c8b45eb11975f936f 2024-11-27T12:25:32,110 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3c3055e4b5609cf4625dfa6b13256b83/C of 3c3055e4b5609cf4625dfa6b13256b83 into e914b2695c21494c8b45eb11975f936f(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T12:25:32,110 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3c3055e4b5609cf4625dfa6b13256b83: 2024-11-27T12:25:32,110 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., storeName=3c3055e4b5609cf4625dfa6b13256b83/C, priority=13, startTime=1732710331605; duration=0sec 2024-11-27T12:25:32,110 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T12:25:32,110 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3c3055e4b5609cf4625dfa6b13256b83:C 2024-11-27T12:25:32,117 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742128_1304 (size=12301) 2024-11-27T12:25:32,118 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=345 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/.tmp/C/9bf6b334ff694e30b39c574c6ad8d938 2024-11-27T12:25:32,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/.tmp/A/50083cd8a6c3484284932c1edcc7fe51 as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/A/50083cd8a6c3484284932c1edcc7fe51 2024-11-27T12:25:32,131 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/A/50083cd8a6c3484284932c1edcc7fe51, entries=150, sequenceid=345, filesize=12.0 K 2024-11-27T12:25:32,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/.tmp/B/1d2ae34d54bf4236912e6107e1c4872b as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/B/1d2ae34d54bf4236912e6107e1c4872b 2024-11-27T12:25:32,135 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/B/1d2ae34d54bf4236912e6107e1c4872b, entries=150, sequenceid=345, filesize=12.0 K 2024-11-27T12:25:32,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/.tmp/C/9bf6b334ff694e30b39c574c6ad8d938 as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/C/9bf6b334ff694e30b39c574c6ad8d938 2024-11-27T12:25:32,141 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/C/9bf6b334ff694e30b39c574c6ad8d938, entries=150, sequenceid=345, filesize=12.0 K 2024-11-27T12:25:32,143 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=0 B/0 for 3c3055e4b5609cf4625dfa6b13256b83 in 521ms, sequenceid=345, compaction requested=false 2024-11-27T12:25:32,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2603): Flush status journal for 3c3055e4b5609cf4625dfa6b13256b83: 2024-11-27T12:25:32,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83. 2024-11-27T12:25:32,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=81 2024-11-27T12:25:32,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36215 {}] master.HMaster(4169): Remote procedure done, pid=81 2024-11-27T12:25:32,146 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=81, resume processing ppid=80 2024-11-27T12:25:32,146 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=81, ppid=80, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4430 sec 2024-11-27T12:25:32,149 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=80, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees in 1.4480 sec 2024-11-27T12:25:32,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(8855): Flush requested on 3c3055e4b5609cf4625dfa6b13256b83 2024-11-27T12:25:32,520 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T12:25:32,521 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3c3055e4b5609cf4625dfa6b13256b83 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-27T12:25:32,535 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/.tmp/A/a8ebb76c9ed14e8bab981095e45c9463 is 50, key is test_row_0/A:col10/1732710332519/Put/seqid=0 2024-11-27T12:25:32,539 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742129_1305 (size=12301) 2024-11-27T12:25:32,540 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=360 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/.tmp/A/a8ebb76c9ed14e8bab981095e45c9463 2024-11-27T12:25:32,547 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/.tmp/B/e1e1c93269d54e7aa4c20d4e0f39cf41 is 50, key is test_row_0/B:col10/1732710332519/Put/seqid=0 2024-11-27T12:25:32,550 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742130_1306 (size=12301) 2024-11-27T12:25:32,558 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:32,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38902 deadline: 1732710392554, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:32,559 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:32,559 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:32,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38978 deadline: 1732710392555, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:32,559 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:32,559 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:32,559 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:32,559 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:32,559 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:32,561 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:32,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38952 deadline: 1732710392558, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:32,562 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:32,562 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:32,562 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:32,561 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:32,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38986 deadline: 1732710392559, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:32,562 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:32,562 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:32,562 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:32,563 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:32,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38944 deadline: 1732710392560, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:32,563 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:32,563 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:32,563 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:32,670 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:32,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38902 deadline: 1732710392668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:32,670 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:32,670 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:32,670 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:32,670 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:32,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38978 deadline: 1732710392668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:32,670 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:32,670 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:32,670 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:32,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38952 deadline: 1732710392668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:32,670 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:32,671 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:32,671 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:32,671 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:32,671 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:32,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38986 deadline: 1732710392668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:32,671 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:32,671 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:32,671 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:32,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38944 deadline: 1732710392668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:32,671 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:32,671 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:32,671 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:32,671 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:32,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-27T12:25:32,849 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T12:25:32,849 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36215 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-27T12:25:32,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36215 {}] procedure2.ProcedureExecutor(1139): Stored pid=82, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees 2024-11-27T12:25:32,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-27T12:25:32,851 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=82, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T12:25:32,852 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=82, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T12:25:32,852 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=83, ppid=82, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T12:25:32,880 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:32,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38902 deadline: 1732710392878, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:32,881 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:32,881 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:32,881 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:32,881 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:32,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38978 deadline: 1732710392878, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:32,881 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:32,881 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:32,881 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:32,881 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:32,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38952 deadline: 1732710392878, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:32,882 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:32,882 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:32,882 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:32,882 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:32,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38986 deadline: 1732710392878, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:32,882 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:32,882 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:32,882 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:32,882 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:32,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38944 deadline: 1732710392878, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:32,882 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:32,882 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:32,882 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:32,951 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=360 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/.tmp/B/e1e1c93269d54e7aa4c20d4e0f39cf41 2024-11-27T12:25:32,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-27T12:25:32,958 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/.tmp/C/03253546e9cd42fc9545f7232ceedbf2 is 50, key is test_row_0/C:col10/1732710332519/Put/seqid=0 2024-11-27T12:25:32,963 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742131_1307 (size=12301) 2024-11-27T12:25:33,004 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-27T12:25:33,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83. 2024-11-27T12:25:33,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83. as already flushing 2024-11-27T12:25:33,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83. 2024-11-27T12:25:33,004 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => 3c3055e4b5609cf4625dfa6b13256b83, NAME => 'TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:25:33,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => 3c3055e4b5609cf4625dfa6b13256b83, NAME => 'TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:25:33,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36215 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3c3055e4b5609cf4625dfa6b13256b83, NAME => 'TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3c3055e4b5609cf4625dfa6b13256b83, NAME => 'TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:25:33,157 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-27T12:25:33,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83. 2024-11-27T12:25:33,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83. as already flushing 2024-11-27T12:25:33,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83. 2024-11-27T12:25:33,157 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => 3c3055e4b5609cf4625dfa6b13256b83, NAME => 'TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:25:33,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => 3c3055e4b5609cf4625dfa6b13256b83, NAME => 'TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:25:33,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36215 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3c3055e4b5609cf4625dfa6b13256b83, NAME => 'TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3c3055e4b5609cf4625dfa6b13256b83, NAME => 'TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:25:33,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-27T12:25:33,190 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:33,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38902 deadline: 1732710393188, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:33,190 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:33,190 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:33,190 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:33,190 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:33,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38978 deadline: 1732710393188, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:33,191 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:33,191 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:33,191 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:33,191 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:33,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38952 deadline: 1732710393188, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:33,191 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:33,191 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:33,191 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:33,191 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:33,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38986 deadline: 1732710393188, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:33,191 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:33,191 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:33,191 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:33,191 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:33,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38944 deadline: 1732710393188, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:33,191 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:33,192 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:33,192 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:33,310 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-27T12:25:33,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83. 2024-11-27T12:25:33,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83. as already flushing 2024-11-27T12:25:33,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83. 2024-11-27T12:25:33,310 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => 3c3055e4b5609cf4625dfa6b13256b83, NAME => 'TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:25:33,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => 3c3055e4b5609cf4625dfa6b13256b83, NAME => 'TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:25:33,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36215 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3c3055e4b5609cf4625dfa6b13256b83, NAME => 'TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3c3055e4b5609cf4625dfa6b13256b83, NAME => 'TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:25:33,364 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=360 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/.tmp/C/03253546e9cd42fc9545f7232ceedbf2 2024-11-27T12:25:33,368 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/.tmp/A/a8ebb76c9ed14e8bab981095e45c9463 as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/A/a8ebb76c9ed14e8bab981095e45c9463 2024-11-27T12:25:33,371 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/A/a8ebb76c9ed14e8bab981095e45c9463, entries=150, sequenceid=360, filesize=12.0 K 2024-11-27T12:25:33,371 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/.tmp/B/e1e1c93269d54e7aa4c20d4e0f39cf41 as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/B/e1e1c93269d54e7aa4c20d4e0f39cf41 2024-11-27T12:25:33,375 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/B/e1e1c93269d54e7aa4c20d4e0f39cf41, entries=150, sequenceid=360, filesize=12.0 K 2024-11-27T12:25:33,375 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/.tmp/C/03253546e9cd42fc9545f7232ceedbf2 as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/C/03253546e9cd42fc9545f7232ceedbf2 2024-11-27T12:25:33,378 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/C/03253546e9cd42fc9545f7232ceedbf2, entries=150, sequenceid=360, filesize=12.0 K 2024-11-27T12:25:33,379 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=147.60 KB/151140 for 3c3055e4b5609cf4625dfa6b13256b83 in 859ms, sequenceid=360, compaction requested=true 2024-11-27T12:25:33,379 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3c3055e4b5609cf4625dfa6b13256b83: 2024-11-27T12:25:33,379 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3c3055e4b5609cf4625dfa6b13256b83:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T12:25:33,379 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T12:25:33,379 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T12:25:33,379 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3c3055e4b5609cf4625dfa6b13256b83:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T12:25:33,379 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T12:25:33,379 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3c3055e4b5609cf4625dfa6b13256b83:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T12:25:33,379 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T12:25:33,379 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T12:25:33,380 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T12:25:33,380 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T12:25:33,380 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HStore(1541): 3c3055e4b5609cf4625dfa6b13256b83/B is initiating minor compaction (all files) 2024-11-27T12:25:33,380 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HStore(1541): 3c3055e4b5609cf4625dfa6b13256b83/A is initiating minor compaction (all files) 2024-11-27T12:25:33,380 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3c3055e4b5609cf4625dfa6b13256b83/A in TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83. 2024-11-27T12:25:33,380 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3c3055e4b5609cf4625dfa6b13256b83/B in TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83. 2024-11-27T12:25:33,380 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/B/060bec5dc8e5470e9040ef4c0f0945a8, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/B/1d2ae34d54bf4236912e6107e1c4872b, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/B/e1e1c93269d54e7aa4c20d4e0f39cf41] into tmpdir=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/.tmp, totalSize=36.8 K 2024-11-27T12:25:33,380 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/A/fa238a787cd7411e81684fcf3a369372, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/A/50083cd8a6c3484284932c1edcc7fe51, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/A/a8ebb76c9ed14e8bab981095e45c9463] into tmpdir=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/.tmp, totalSize=36.8 K 2024-11-27T12:25:33,381 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.Compactor(225): Compacting 060bec5dc8e5470e9040ef4c0f0945a8, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=320, earliestPutTs=1732710330326 2024-11-27T12:25:33,381 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] compactions.Compactor(225): Compacting fa238a787cd7411e81684fcf3a369372, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=320, earliestPutTs=1732710330326 2024-11-27T12:25:33,381 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.Compactor(225): Compacting 1d2ae34d54bf4236912e6107e1c4872b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=345, earliestPutTs=1732710330349 2024-11-27T12:25:33,381 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] compactions.Compactor(225): Compacting 50083cd8a6c3484284932c1edcc7fe51, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=345, earliestPutTs=1732710330349 2024-11-27T12:25:33,381 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.Compactor(225): Compacting e1e1c93269d54e7aa4c20d4e0f39cf41, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=360, earliestPutTs=1732710332518 2024-11-27T12:25:33,381 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] compactions.Compactor(225): Compacting a8ebb76c9ed14e8bab981095e45c9463, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=360, earliestPutTs=1732710332518 2024-11-27T12:25:33,388 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3c3055e4b5609cf4625dfa6b13256b83#B#compaction#261 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-27T12:25:33,388 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/.tmp/B/4148add751e4419481b20abf006fe6b4 is 50, key is test_row_0/B:col10/1732710332519/Put/seqid=0 2024-11-27T12:25:33,389 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3c3055e4b5609cf4625dfa6b13256b83#A#compaction#262 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T12:25:33,389 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/.tmp/A/985de2a009aa43ed9c5e3211e09ddf1a is 50, key is test_row_0/A:col10/1732710332519/Put/seqid=0 2024-11-27T12:25:33,401 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742133_1309 (size=13153) 2024-11-27T12:25:33,405 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742132_1308 (size=13153) 2024-11-27T12:25:33,462 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-27T12:25:33,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83. 2024-11-27T12:25:33,463 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2902): Flushing 3c3055e4b5609cf4625dfa6b13256b83 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-27T12:25:33,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/.tmp/A/868a6b712e4d47dd877151e825f2f056 is 50, key is test_row_0/A:col10/1732710332558/Put/seqid=0 2024-11-27T12:25:33,471 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742134_1310 (size=12301) 2024-11-27T12:25:33,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-27T12:25:33,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(8855): Flush requested on 3c3055e4b5609cf4625dfa6b13256b83 2024-11-27T12:25:33,700 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83. as already flushing 2024-11-27T12:25:33,706 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:33,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38944 deadline: 1732710393703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:33,707 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:33,707 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:33,707 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:33,708 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:33,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38902 deadline: 1732710393705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:33,709 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:33,709 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:33,709 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:33,710 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:33,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38978 deadline: 1732710393705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:33,710 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:33,710 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:33,710 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:33,710 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:33,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38952 deadline: 1732710393706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:33,711 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:33,711 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:33,711 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:33,711 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:33,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38986 deadline: 1732710393706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:33,711 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:33,711 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:33,711 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:33,806 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/.tmp/B/4148add751e4419481b20abf006fe6b4 as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/B/4148add751e4419481b20abf006fe6b4 2024-11-27T12:25:33,809 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/.tmp/A/985de2a009aa43ed9c5e3211e09ddf1a as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/A/985de2a009aa43ed9c5e3211e09ddf1a 2024-11-27T12:25:33,809 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3c3055e4b5609cf4625dfa6b13256b83/B of 3c3055e4b5609cf4625dfa6b13256b83 into 4148add751e4419481b20abf006fe6b4(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T12:25:33,809 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3c3055e4b5609cf4625dfa6b13256b83: 2024-11-27T12:25:33,809 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., storeName=3c3055e4b5609cf4625dfa6b13256b83/B, priority=13, startTime=1732710333379; duration=0sec 2024-11-27T12:25:33,810 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T12:25:33,810 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3c3055e4b5609cf4625dfa6b13256b83:B 2024-11-27T12:25:33,810 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T12:25:33,810 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T12:25:33,810 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HStore(1541): 3c3055e4b5609cf4625dfa6b13256b83/C is initiating minor compaction (all files) 2024-11-27T12:25:33,810 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3c3055e4b5609cf4625dfa6b13256b83/C in TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83. 2024-11-27T12:25:33,811 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/C/e914b2695c21494c8b45eb11975f936f, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/C/9bf6b334ff694e30b39c574c6ad8d938, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/C/03253546e9cd42fc9545f7232ceedbf2] into tmpdir=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/.tmp, totalSize=36.8 K 2024-11-27T12:25:33,811 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.Compactor(225): Compacting e914b2695c21494c8b45eb11975f936f, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=320, earliestPutTs=1732710330326 2024-11-27T12:25:33,811 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.Compactor(225): Compacting 9bf6b334ff694e30b39c574c6ad8d938, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=345, earliestPutTs=1732710330349 2024-11-27T12:25:33,811 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.Compactor(225): Compacting 03253546e9cd42fc9545f7232ceedbf2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=360, earliestPutTs=1732710332518 2024-11-27T12:25:33,814 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3c3055e4b5609cf4625dfa6b13256b83/A of 3c3055e4b5609cf4625dfa6b13256b83 into 985de2a009aa43ed9c5e3211e09ddf1a(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T12:25:33,814 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3c3055e4b5609cf4625dfa6b13256b83: 2024-11-27T12:25:33,814 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., storeName=3c3055e4b5609cf4625dfa6b13256b83/A, priority=13, startTime=1732710333379; duration=0sec 2024-11-27T12:25:33,814 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T12:25:33,814 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3c3055e4b5609cf4625dfa6b13256b83:A 2024-11-27T12:25:33,818 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3c3055e4b5609cf4625dfa6b13256b83#C#compaction#264 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T12:25:33,819 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/.tmp/C/9c4220e9e36e4dd594ba60f8fca8ce02 is 50, key is test_row_0/C:col10/1732710332519/Put/seqid=0 2024-11-27T12:25:33,820 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:33,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38944 deadline: 1732710393818, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:33,820 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:33,820 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:33,821 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:33,821 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:33,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38978 deadline: 1732710393819, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:33,821 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:33,821 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:33,821 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:33,821 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:33,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38902 deadline: 1732710393819, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:33,821 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:33,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38952 deadline: 1732710393819, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:33,821 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:33,821 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:33,821 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:33,821 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:33,822 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:33,822 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:33,822 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:33,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38986 deadline: 1732710393819, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:33,822 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:33,822 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:33,822 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:33,833 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742135_1311 (size=13153) 2024-11-27T12:25:33,872 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=385 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/.tmp/A/868a6b712e4d47dd877151e825f2f056 2024-11-27T12:25:33,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/.tmp/B/3fc3fe09dd474da1b979205b37affabc is 50, key is test_row_0/B:col10/1732710332558/Put/seqid=0 2024-11-27T12:25:33,882 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742136_1312 (size=12301) 2024-11-27T12:25:33,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-27T12:25:34,030 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:34,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38944 deadline: 1732710394027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:34,030 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:34,030 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:34,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38978 deadline: 1732710394028, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:34,030 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:34,030 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:34,030 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:34,030 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:34,030 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:34,030 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:34,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38902 deadline: 1732710394028, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:34,031 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:34,031 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:34,031 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:34,031 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:34,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38952 deadline: 1732710394028, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:34,031 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:34,031 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:34,031 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:34,031 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:34,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38986 deadline: 1732710394028, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:34,031 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:34,031 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:34,031 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:34,237 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/.tmp/C/9c4220e9e36e4dd594ba60f8fca8ce02 as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/C/9c4220e9e36e4dd594ba60f8fca8ce02 2024-11-27T12:25:34,241 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3c3055e4b5609cf4625dfa6b13256b83/C of 3c3055e4b5609cf4625dfa6b13256b83 into 9c4220e9e36e4dd594ba60f8fca8ce02(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T12:25:34,241 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3c3055e4b5609cf4625dfa6b13256b83: 2024-11-27T12:25:34,241 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., storeName=3c3055e4b5609cf4625dfa6b13256b83/C, priority=13, startTime=1732710333379; duration=0sec 2024-11-27T12:25:34,242 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T12:25:34,242 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3c3055e4b5609cf4625dfa6b13256b83:C 2024-11-27T12:25:34,283 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=385 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/.tmp/B/3fc3fe09dd474da1b979205b37affabc 2024-11-27T12:25:34,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/.tmp/C/592c6ed2afde456b8b71e2978e863715 is 50, key is test_row_0/C:col10/1732710332558/Put/seqid=0 2024-11-27T12:25:34,292 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742137_1313 (size=12301) 2024-11-27T12:25:34,340 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:34,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38944 deadline: 1732710394338, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:34,340 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:34,340 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:34,340 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:34,341 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:34,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38978 deadline: 1732710394338, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:34,341 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:34,341 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:34,341 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:34,341 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:34,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38902 deadline: 1732710394339, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:34,341 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:34,341 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:34,341 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:34,341 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:34,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38952 deadline: 1732710394339, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:34,341 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:34,342 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:34,342 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:34,342 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:34,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38986 deadline: 1732710394339, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:34,342 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:34,342 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:34,342 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:34,693 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=385 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/.tmp/C/592c6ed2afde456b8b71e2978e863715 2024-11-27T12:25:34,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/.tmp/A/868a6b712e4d47dd877151e825f2f056 as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/A/868a6b712e4d47dd877151e825f2f056 2024-11-27T12:25:34,702 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/A/868a6b712e4d47dd877151e825f2f056, entries=150, sequenceid=385, filesize=12.0 K 2024-11-27T12:25:34,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/.tmp/B/3fc3fe09dd474da1b979205b37affabc as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/B/3fc3fe09dd474da1b979205b37affabc 2024-11-27T12:25:34,706 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/B/3fc3fe09dd474da1b979205b37affabc, entries=150, sequenceid=385, filesize=12.0 K 2024-11-27T12:25:34,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/.tmp/C/592c6ed2afde456b8b71e2978e863715 as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/C/592c6ed2afde456b8b71e2978e863715 2024-11-27T12:25:34,711 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/C/592c6ed2afde456b8b71e2978e863715, entries=150, sequenceid=385, filesize=12.0 K 2024-11-27T12:25:34,711 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=60.38 KB/61830 for 3c3055e4b5609cf4625dfa6b13256b83 in 1248ms, sequenceid=385, compaction requested=false 2024-11-27T12:25:34,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2603): Flush status journal for 3c3055e4b5609cf4625dfa6b13256b83: 2024-11-27T12:25:34,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83. 2024-11-27T12:25:34,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=83 2024-11-27T12:25:34,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36215 {}] master.HMaster(4169): Remote procedure done, pid=83 2024-11-27T12:25:34,715 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=83, resume processing ppid=82 2024-11-27T12:25:34,715 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=83, ppid=82, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8610 sec 2024-11-27T12:25:34,716 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=82, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees in 1.8660 sec 2024-11-27T12:25:34,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(8855): Flush requested on 3c3055e4b5609cf4625dfa6b13256b83 2024-11-27T12:25:34,850 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T12:25:34,850 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3c3055e4b5609cf4625dfa6b13256b83 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-27T12:25:34,856 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/.tmp/A/26872716fb0d4608bfeeff0d231a1303 is 50, key is test_row_0/A:col10/1732710333705/Put/seqid=0 2024-11-27T12:25:34,861 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742138_1314 (size=12301) 2024-11-27T12:25:34,873 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:34,873 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:34,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38902 deadline: 1732710394869, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:34,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38978 deadline: 1732710394868, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:34,874 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:34,874 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:34,874 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:34,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38952 deadline: 1732710394869, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:34,874 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:34,874 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:34,874 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:34,874 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:34,874 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:34,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38986 deadline: 1732710394870, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:34,874 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:34,874 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:34,874 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:34,874 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:34,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38944 deadline: 1732710394871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:34,874 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:34,874 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:34,874 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:34,874 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:34,874 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:34,875 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:34,980 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:34,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38902 deadline: 1732710394978, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:34,981 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:34,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38978 deadline: 1732710394978, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:34,981 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:34,981 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:34,981 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:34,981 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:34,981 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:34,981 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:34,981 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:34,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38952 deadline: 1732710394979, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:34,981 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:34,981 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:34,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38986 deadline: 1732710394979, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:34,981 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:34,981 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:34,981 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:34,981 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:34,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38944 deadline: 1732710394979, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:34,981 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:34,982 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:34,982 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:34,982 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:34,982 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:34,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-27T12:25:34,998 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T12:25:34,999 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36215 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-27T12:25:35,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36215 {}] procedure2.ProcedureExecutor(1139): Stored pid=84, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=84, table=TestAcidGuarantees 2024-11-27T12:25:35,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-27T12:25:35,000 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=84, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=84, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T12:25:35,001 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=84, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=84, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T12:25:35,001 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=85, ppid=84, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T12:25:35,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-27T12:25:35,153 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-11-27T12:25:35,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83. 2024-11-27T12:25:35,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83. as already flushing 2024-11-27T12:25:35,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83. 2024-11-27T12:25:35,153 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] handler.RSProcedureHandler(58): pid=85 java.io.IOException: Unable to complete flush {ENCODED => 3c3055e4b5609cf4625dfa6b13256b83, NAME => 'TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:25:35,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=85 java.io.IOException: Unable to complete flush {ENCODED => 3c3055e4b5609cf4625dfa6b13256b83, NAME => 'TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:25:35,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36215 {}] master.HMaster(4177): Remote procedure failed, pid=85 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3c3055e4b5609cf4625dfa6b13256b83, NAME => 'TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3c3055e4b5609cf4625dfa6b13256b83, NAME => 'TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:25:35,190 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:35,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38902 deadline: 1732710395188, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:35,191 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:35,191 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:35,191 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:35,191 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:35,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38978 deadline: 1732710395188, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:35,191 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:35,191 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:35,191 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:35,191 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:35,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38952 deadline: 1732710395188, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:35,191 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:35,191 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:35,191 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:35,191 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:35,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38986 deadline: 1732710395188, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:35,192 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:35,192 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:35,192 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:35,192 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:35,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38944 deadline: 1732710395188, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:35,192 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:35,192 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:35,192 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:35,263 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=402 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/.tmp/A/26872716fb0d4608bfeeff0d231a1303 2024-11-27T12:25:35,269 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/.tmp/B/1a0d050a9f944ab08dbc1ce632f0f47d is 50, key is test_row_0/B:col10/1732710333705/Put/seqid=0 2024-11-27T12:25:35,273 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742139_1315 (size=12301) 2024-11-27T12:25:35,306 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-11-27T12:25:35,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83. 2024-11-27T12:25:35,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83. as already flushing 2024-11-27T12:25:35,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83. 2024-11-27T12:25:35,306 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] handler.RSProcedureHandler(58): pid=85 java.io.IOException: Unable to complete flush {ENCODED => 3c3055e4b5609cf4625dfa6b13256b83, NAME => 'TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:25:35,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=85 java.io.IOException: Unable to complete flush {ENCODED => 3c3055e4b5609cf4625dfa6b13256b83, NAME => 'TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:25:35,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36215 {}] master.HMaster(4177): Remote procedure failed, pid=85 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3c3055e4b5609cf4625dfa6b13256b83, NAME => 'TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3c3055e4b5609cf4625dfa6b13256b83, NAME => 'TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:25:35,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-27T12:25:35,458 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-11-27T12:25:35,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83. 2024-11-27T12:25:35,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83. as already flushing 2024-11-27T12:25:35,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83. 2024-11-27T12:25:35,459 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] handler.RSProcedureHandler(58): pid=85 java.io.IOException: Unable to complete flush {ENCODED => 3c3055e4b5609cf4625dfa6b13256b83, NAME => 'TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:25:35,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=85 java.io.IOException: Unable to complete flush {ENCODED => 3c3055e4b5609cf4625dfa6b13256b83, NAME => 'TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:25:35,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36215 {}] master.HMaster(4177): Remote procedure failed, pid=85 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3c3055e4b5609cf4625dfa6b13256b83, NAME => 'TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3c3055e4b5609cf4625dfa6b13256b83, NAME => 'TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:25:35,500 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:35,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38902 deadline: 1732710395498, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:35,500 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:35,500 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:35,501 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:35,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38978 deadline: 1732710395498, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:35,501 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:35,501 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:35,501 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:35,501 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:35,501 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:35,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38952 deadline: 1732710395498, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:35,501 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:35,501 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:35,501 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:35,501 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:35,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38986 deadline: 1732710395498, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:35,501 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:35,501 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:35,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38944 deadline: 1732710395498, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:35,501 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:35,501 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:35,502 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:35,502 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:35,502 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:35,611 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-11-27T12:25:35,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83. 2024-11-27T12:25:35,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83. as already flushing 2024-11-27T12:25:35,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83. 2024-11-27T12:25:35,612 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] handler.RSProcedureHandler(58): pid=85 java.io.IOException: Unable to complete flush {ENCODED => 3c3055e4b5609cf4625dfa6b13256b83, NAME => 'TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:25:35,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=85 java.io.IOException: Unable to complete flush {ENCODED => 3c3055e4b5609cf4625dfa6b13256b83, NAME => 'TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:25:35,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36215 {}] master.HMaster(4177): Remote procedure failed, pid=85 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3c3055e4b5609cf4625dfa6b13256b83, NAME => 'TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3c3055e4b5609cf4625dfa6b13256b83, NAME => 'TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:25:35,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-27T12:25:35,674 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=402 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/.tmp/B/1a0d050a9f944ab08dbc1ce632f0f47d 2024-11-27T12:25:35,686 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/.tmp/C/c8e569b4d5034a31a22dcfd101e7b6e7 is 50, key is test_row_0/C:col10/1732710333705/Put/seqid=0 2024-11-27T12:25:35,689 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742140_1316 (size=12301) 2024-11-27T12:25:35,764 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-11-27T12:25:35,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83. 2024-11-27T12:25:35,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83. as already flushing 2024-11-27T12:25:35,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83. 2024-11-27T12:25:35,764 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] handler.RSProcedureHandler(58): pid=85 java.io.IOException: Unable to complete flush {ENCODED => 3c3055e4b5609cf4625dfa6b13256b83, NAME => 'TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:25:35,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=85 java.io.IOException: Unable to complete flush {ENCODED => 3c3055e4b5609cf4625dfa6b13256b83, NAME => 'TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:25:35,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36215 {}] master.HMaster(4177): Remote procedure failed, pid=85 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3c3055e4b5609cf4625dfa6b13256b83, NAME => 'TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3c3055e4b5609cf4625dfa6b13256b83, NAME => 'TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:25:35,917 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-11-27T12:25:35,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83. 2024-11-27T12:25:35,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83. as already flushing 2024-11-27T12:25:35,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83. 2024-11-27T12:25:35,917 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] handler.RSProcedureHandler(58): pid=85 java.io.IOException: Unable to complete flush {ENCODED => 3c3055e4b5609cf4625dfa6b13256b83, NAME => 'TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:25:35,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=85 java.io.IOException: Unable to complete flush {ENCODED => 3c3055e4b5609cf4625dfa6b13256b83, NAME => 'TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:25:35,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36215 {}] master.HMaster(4177): Remote procedure failed, pid=85 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3c3055e4b5609cf4625dfa6b13256b83, NAME => 'TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3c3055e4b5609cf4625dfa6b13256b83, NAME => 'TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:25:36,011 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:36,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38902 deadline: 1732710396008, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:36,011 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:36,011 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:36,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38986 deadline: 1732710396008, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:36,011 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:36,011 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:36,011 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:36,011 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:36,011 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:36,011 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:36,011 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:36,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38978 deadline: 1732710396009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:36,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38952 deadline: 1732710396008, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:36,011 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:36,011 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:36,011 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:36,011 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:36,011 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:36,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38944 deadline: 1732710396009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:36,011 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:36,012 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:36,012 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:36,012 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:36,012 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:25:36,069 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-11-27T12:25:36,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83. 2024-11-27T12:25:36,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83. as already flushing 2024-11-27T12:25:36,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83. 2024-11-27T12:25:36,070 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] handler.RSProcedureHandler(58): pid=85 java.io.IOException: Unable to complete flush {ENCODED => 3c3055e4b5609cf4625dfa6b13256b83, NAME => 'TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:25:36,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=85 java.io.IOException: Unable to complete flush {ENCODED => 3c3055e4b5609cf4625dfa6b13256b83, NAME => 'TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:25:36,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36215 {}] master.HMaster(4177): Remote procedure failed, pid=85 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3c3055e4b5609cf4625dfa6b13256b83, NAME => 'TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3c3055e4b5609cf4625dfa6b13256b83, NAME => 'TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:25:36,090 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=402 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/.tmp/C/c8e569b4d5034a31a22dcfd101e7b6e7 2024-11-27T12:25:36,094 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/.tmp/A/26872716fb0d4608bfeeff0d231a1303 as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/A/26872716fb0d4608bfeeff0d231a1303 2024-11-27T12:25:36,097 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/A/26872716fb0d4608bfeeff0d231a1303, entries=150, sequenceid=402, filesize=12.0 K 2024-11-27T12:25:36,097 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/.tmp/B/1a0d050a9f944ab08dbc1ce632f0f47d as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/B/1a0d050a9f944ab08dbc1ce632f0f47d 2024-11-27T12:25:36,101 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/B/1a0d050a9f944ab08dbc1ce632f0f47d, entries=150, sequenceid=402, filesize=12.0 K 2024-11-27T12:25:36,101 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/.tmp/C/c8e569b4d5034a31a22dcfd101e7b6e7 as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/C/c8e569b4d5034a31a22dcfd101e7b6e7 2024-11-27T12:25:36,104 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/C/c8e569b4d5034a31a22dcfd101e7b6e7, entries=150, sequenceid=402, filesize=12.0 K 2024-11-27T12:25:36,105 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for 3c3055e4b5609cf4625dfa6b13256b83 in 1255ms, sequenceid=402, compaction requested=true 2024-11-27T12:25:36,105 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3c3055e4b5609cf4625dfa6b13256b83: 2024-11-27T12:25:36,105 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3c3055e4b5609cf4625dfa6b13256b83:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T12:25:36,105 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T12:25:36,105 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T12:25:36,106 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3c3055e4b5609cf4625dfa6b13256b83:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T12:25:36,106 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T12:25:36,106 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T12:25:36,106 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3c3055e4b5609cf4625dfa6b13256b83:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T12:25:36,106 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T12:25:36,106 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T12:25:36,106 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T12:25:36,106 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HStore(1541): 3c3055e4b5609cf4625dfa6b13256b83/B is initiating minor compaction (all files) 2024-11-27T12:25:36,106 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HStore(1541): 3c3055e4b5609cf4625dfa6b13256b83/A is initiating minor compaction (all files) 2024-11-27T12:25:36,106 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3c3055e4b5609cf4625dfa6b13256b83/B in TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83. 2024-11-27T12:25:36,106 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3c3055e4b5609cf4625dfa6b13256b83/A in TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83. 2024-11-27T12:25:36,106 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/A/985de2a009aa43ed9c5e3211e09ddf1a, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/A/868a6b712e4d47dd877151e825f2f056, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/A/26872716fb0d4608bfeeff0d231a1303] into tmpdir=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/.tmp, totalSize=36.9 K 2024-11-27T12:25:36,106 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/B/4148add751e4419481b20abf006fe6b4, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/B/3fc3fe09dd474da1b979205b37affabc, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/B/1a0d050a9f944ab08dbc1ce632f0f47d] into tmpdir=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/.tmp, totalSize=36.9 K 2024-11-27T12:25:36,107 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] compactions.Compactor(225): Compacting 985de2a009aa43ed9c5e3211e09ddf1a, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=360, earliestPutTs=1732710332518 2024-11-27T12:25:36,107 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.Compactor(225): Compacting 4148add751e4419481b20abf006fe6b4, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=360, earliestPutTs=1732710332518 2024-11-27T12:25:36,107 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] compactions.Compactor(225): Compacting 868a6b712e4d47dd877151e825f2f056, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=385, earliestPutTs=1732710332550 2024-11-27T12:25:36,107 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.Compactor(225): Compacting 3fc3fe09dd474da1b979205b37affabc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=385, earliestPutTs=1732710332550 2024-11-27T12:25:36,107 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] compactions.Compactor(225): Compacting 26872716fb0d4608bfeeff0d231a1303, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=402, earliestPutTs=1732710333705 2024-11-27T12:25:36,107 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.Compactor(225): Compacting 1a0d050a9f944ab08dbc1ce632f0f47d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=402, earliestPutTs=1732710333705 2024-11-27T12:25:36,114 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3c3055e4b5609cf4625dfa6b13256b83#B#compaction#270 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-27T12:25:36,114 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3c3055e4b5609cf4625dfa6b13256b83#A#compaction#271 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T12:25:36,114 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/.tmp/A/aee3313394af49f18f941aa9790176fb is 50, key is test_row_0/A:col10/1732710333705/Put/seqid=0 2024-11-27T12:25:36,114 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/.tmp/B/7604d4059638431683177b07f606ee25 is 50, key is test_row_0/B:col10/1732710333705/Put/seqid=0 2024-11-27T12:25:36,129 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742142_1318 (size=13255) 2024-11-27T12:25:36,129 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742141_1317 (size=13255) 2024-11-27T12:25:36,134 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/.tmp/B/7604d4059638431683177b07f606ee25 as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/B/7604d4059638431683177b07f606ee25 2024-11-27T12:25:36,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-27T12:25:36,139 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3c3055e4b5609cf4625dfa6b13256b83/B of 3c3055e4b5609cf4625dfa6b13256b83 into 7604d4059638431683177b07f606ee25(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T12:25:36,139 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3c3055e4b5609cf4625dfa6b13256b83: 2024-11-27T12:25:36,139 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., storeName=3c3055e4b5609cf4625dfa6b13256b83/B, priority=13, startTime=1732710336105; duration=0sec 2024-11-27T12:25:36,139 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T12:25:36,139 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3c3055e4b5609cf4625dfa6b13256b83:B 2024-11-27T12:25:36,139 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T12:25:36,140 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T12:25:36,140 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HStore(1541): 3c3055e4b5609cf4625dfa6b13256b83/C is initiating minor compaction (all files) 2024-11-27T12:25:36,140 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3c3055e4b5609cf4625dfa6b13256b83/C in TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83. 2024-11-27T12:25:36,140 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/C/9c4220e9e36e4dd594ba60f8fca8ce02, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/C/592c6ed2afde456b8b71e2978e863715, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/C/c8e569b4d5034a31a22dcfd101e7b6e7] into tmpdir=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/.tmp, totalSize=36.9 K 2024-11-27T12:25:36,141 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.Compactor(225): Compacting 9c4220e9e36e4dd594ba60f8fca8ce02, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=360, earliestPutTs=1732710332518 2024-11-27T12:25:36,141 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.Compactor(225): Compacting 592c6ed2afde456b8b71e2978e863715, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=385, earliestPutTs=1732710332550 2024-11-27T12:25:36,141 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.Compactor(225): Compacting c8e569b4d5034a31a22dcfd101e7b6e7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=402, earliestPutTs=1732710333705 2024-11-27T12:25:36,147 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3c3055e4b5609cf4625dfa6b13256b83#C#compaction#272 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T12:25:36,147 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/.tmp/C/8bd691e8afd7491eaa572d0edf1a85ab is 50, key is test_row_0/C:col10/1732710333705/Put/seqid=0 2024-11-27T12:25:36,153 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742143_1319 (size=13255) 2024-11-27T12:25:36,160 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/.tmp/C/8bd691e8afd7491eaa572d0edf1a85ab as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/C/8bd691e8afd7491eaa572d0edf1a85ab 2024-11-27T12:25:36,165 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3c3055e4b5609cf4625dfa6b13256b83/C of 3c3055e4b5609cf4625dfa6b13256b83 into 8bd691e8afd7491eaa572d0edf1a85ab(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T12:25:36,165 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3c3055e4b5609cf4625dfa6b13256b83: 2024-11-27T12:25:36,165 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., storeName=3c3055e4b5609cf4625dfa6b13256b83/C, priority=13, startTime=1732710336106; duration=0sec 2024-11-27T12:25:36,165 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T12:25:36,165 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3c3055e4b5609cf4625dfa6b13256b83:C 2024-11-27T12:25:36,222 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-11-27T12:25:36,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83. 2024-11-27T12:25:36,223 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2902): Flushing 3c3055e4b5609cf4625dfa6b13256b83 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-27T12:25:36,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/.tmp/A/dd0427338e7e4e6a876beb14e46d5792 is 50, key is test_row_0/A:col10/1732710334867/Put/seqid=0 2024-11-27T12:25:36,232 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742144_1320 (size=12301) 2024-11-27T12:25:36,533 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/.tmp/A/aee3313394af49f18f941aa9790176fb as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/A/aee3313394af49f18f941aa9790176fb 2024-11-27T12:25:36,538 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3c3055e4b5609cf4625dfa6b13256b83/A of 3c3055e4b5609cf4625dfa6b13256b83 into aee3313394af49f18f941aa9790176fb(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T12:25:36,538 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3c3055e4b5609cf4625dfa6b13256b83: 2024-11-27T12:25:36,538 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., storeName=3c3055e4b5609cf4625dfa6b13256b83/A, priority=13, startTime=1732710336105; duration=0sec 2024-11-27T12:25:36,538 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T12:25:36,538 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3c3055e4b5609cf4625dfa6b13256b83:A 2024-11-27T12:25:36,633 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=426 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/.tmp/A/dd0427338e7e4e6a876beb14e46d5792 2024-11-27T12:25:36,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/.tmp/B/f1a18069e2b746739d48b00cd08c1906 is 50, key is test_row_0/B:col10/1732710334867/Put/seqid=0 2024-11-27T12:25:36,674 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742145_1321 (size=12301) 2024-11-27T12:25:36,913 INFO [Thread-1099 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1099. 2024-11-27T12:25:36,913 DEBUG [Thread-1099 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T12:25:36,913 DEBUG [Thread-1099 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T12:25:36,913 DEBUG [Thread-1099 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T12:25:36,913 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T12:25:36,913 INFO [Thread-1103 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1103. 2024-11-27T12:25:36,913 DEBUG [Thread-1103 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T12:25:36,913 DEBUG [Thread-1103 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T12:25:36,913 DEBUG [Thread-1103 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T12:25:36,913 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T12:25:36,914 INFO [Thread-1107 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1107. 2024-11-27T12:25:36,914 DEBUG [Thread-1107 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T12:25:36,914 DEBUG [Thread-1107 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T12:25:36,914 DEBUG [Thread-1107 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T12:25:36,914 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T12:25:36,915 INFO [Thread-1105 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1105. 2024-11-27T12:25:36,915 DEBUG [Thread-1105 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T12:25:36,915 DEBUG [Thread-1105 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T12:25:36,915 DEBUG [Thread-1105 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T12:25:36,915 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T12:25:36,915 INFO [Thread-1101 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1101. 2024-11-27T12:25:36,915 DEBUG [Thread-1101 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T12:25:36,915 DEBUG [Thread-1101 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T12:25:36,915 DEBUG [Thread-1101 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T12:25:36,915 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T12:25:37,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(8855): Flush requested on 3c3055e4b5609cf4625dfa6b13256b83 2024-11-27T12:25:37,019 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83. as already flushing 2024-11-27T12:25:37,019 INFO [Thread-1088 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1088. 2024-11-27T12:25:37,019 INFO [Thread-1096 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1096. 2024-11-27T12:25:37,020 DEBUG [Thread-1088 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T12:25:37,020 DEBUG [Thread-1096 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T12:25:37,020 DEBUG [Thread-1088 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T12:25:37,020 DEBUG [Thread-1096 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T12:25:37,020 DEBUG [Thread-1088 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T12:25:37,020 DEBUG [Thread-1096 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T12:25:37,020 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T12:25:37,020 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T12:25:37,031 INFO [Thread-1094 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1094. 2024-11-27T12:25:37,031 DEBUG [Thread-1094 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T12:25:37,031 INFO [Thread-1092 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1092. 2024-11-27T12:25:37,031 INFO [Thread-1090 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1090. 2024-11-27T12:25:37,031 DEBUG [Thread-1094 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T12:25:37,031 DEBUG [Thread-1092 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T12:25:37,031 DEBUG [Thread-1090 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T12:25:37,031 DEBUG [Thread-1092 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T12:25:37,031 DEBUG [Thread-1090 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T12:25:37,031 DEBUG [Thread-1094 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T12:25:37,031 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T12:25:37,031 DEBUG [Thread-1092 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T12:25:37,031 DEBUG [Thread-1090 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T12:25:37,031 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T12:25:37,032 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T12:25:37,076 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=426 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/.tmp/B/f1a18069e2b746739d48b00cd08c1906 2024-11-27T12:25:37,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/.tmp/C/55df93d09b9e464487348302ea9f3e59 is 50, key is test_row_0/C:col10/1732710334867/Put/seqid=0 2024-11-27T12:25:37,093 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742146_1322 (size=12301) 2024-11-27T12:25:37,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-27T12:25:37,494 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=426 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/.tmp/C/55df93d09b9e464487348302ea9f3e59 2024-11-27T12:25:37,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/.tmp/A/dd0427338e7e4e6a876beb14e46d5792 as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/A/dd0427338e7e4e6a876beb14e46d5792 2024-11-27T12:25:37,503 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/A/dd0427338e7e4e6a876beb14e46d5792, entries=150, sequenceid=426, filesize=12.0 K 2024-11-27T12:25:37,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/.tmp/B/f1a18069e2b746739d48b00cd08c1906 as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/B/f1a18069e2b746739d48b00cd08c1906 2024-11-27T12:25:37,509 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/B/f1a18069e2b746739d48b00cd08c1906, entries=150, sequenceid=426, filesize=12.0 K 2024-11-27T12:25:37,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/.tmp/C/55df93d09b9e464487348302ea9f3e59 as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/C/55df93d09b9e464487348302ea9f3e59 2024-11-27T12:25:37,514 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/C/55df93d09b9e464487348302ea9f3e59, entries=150, sequenceid=426, filesize=12.0 K 2024-11-27T12:25:37,515 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=33.54 KB/34350 for 3c3055e4b5609cf4625dfa6b13256b83 in 1293ms, sequenceid=426, compaction requested=false 2024-11-27T12:25:37,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2603): Flush status journal for 3c3055e4b5609cf4625dfa6b13256b83: 2024-11-27T12:25:37,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83. 2024-11-27T12:25:37,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=85 2024-11-27T12:25:37,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36215 {}] master.HMaster(4169): Remote procedure done, pid=85 2024-11-27T12:25:37,520 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=85, resume processing ppid=84 2024-11-27T12:25:37,520 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=85, ppid=84, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.5160 sec 2024-11-27T12:25:37,522 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=84, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=84, table=TestAcidGuarantees in 2.5210 sec 2024-11-27T12:25:39,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-27T12:25:39,159 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T12:25:39,159 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-27T12:25:39,159 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 72 2024-11-27T12:25:39,159 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 68 2024-11-27T12:25:39,159 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 58 2024-11-27T12:25:39,159 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 73 2024-11-27T12:25:39,159 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 70 2024-11-27T12:25:39,159 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-27T12:25:39,159 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3674 2024-11-27T12:25:39,159 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3588 2024-11-27T12:25:39,159 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3593 2024-11-27T12:25:39,159 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3643 2024-11-27T12:25:39,159 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3603 2024-11-27T12:25:39,159 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-27T12:25:39,160 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-27T12:25:39,160 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testGetAtomicity(AcidGuaranteesTestBase.java:107) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-27T12:25:39,160 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T12:25:39,160 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T12:25:39,160 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T12:25:39,162 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36215 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-11-27T12:25:39,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36215 {}] procedure2.ProcedureExecutor(1139): Stored pid=86, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-27T12:25:39,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-27T12:25:39,167 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732710339167"}]},"ts":"1732710339167"} 2024-11-27T12:25:39,169 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-27T12:25:39,169 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-27T12:25:39,170 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=87, ppid=86, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-27T12:25:39,172 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=3c3055e4b5609cf4625dfa6b13256b83, UNASSIGN}] 2024-11-27T12:25:39,173 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=3c3055e4b5609cf4625dfa6b13256b83, UNASSIGN 2024-11-27T12:25:39,173 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=88 updating hbase:meta row=3c3055e4b5609cf4625dfa6b13256b83, regionState=CLOSING, regionLocation=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:39,174 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=3c3055e4b5609cf4625dfa6b13256b83, UNASSIGN because future has completed 2024-11-27T12:25:39,175 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-27T12:25:39,175 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=89, ppid=88, state=RUNNABLE, hasLock=false; CloseRegionProcedure 3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631}] 2024-11-27T12:25:39,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-27T12:25:39,328 INFO [RS_CLOSE_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] handler.UnassignRegionHandler(122): Close 3c3055e4b5609cf4625dfa6b13256b83 2024-11-27T12:25:39,328 DEBUG [RS_CLOSE_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-27T12:25:39,329 DEBUG [RS_CLOSE_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(1722): Closing 3c3055e4b5609cf4625dfa6b13256b83, disabling compactions & flushes 2024-11-27T12:25:39,329 INFO [RS_CLOSE_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83. 2024-11-27T12:25:39,329 DEBUG [RS_CLOSE_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83. 2024-11-27T12:25:39,329 DEBUG [RS_CLOSE_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83. after waiting 0 ms 2024-11-27T12:25:39,329 DEBUG [RS_CLOSE_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83. 2024-11-27T12:25:39,329 INFO [RS_CLOSE_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(2902): Flushing 3c3055e4b5609cf4625dfa6b13256b83 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-11-27T12:25:39,337 DEBUG [RS_CLOSE_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/.tmp/A/58de7bc062b04296bd7e6b4549827054 is 50, key is test_row_0/A:col10/1732710337029/Put/seqid=0 2024-11-27T12:25:39,341 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742147_1323 (size=9857) 2024-11-27T12:25:39,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-27T12:25:39,743 INFO [RS_CLOSE_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=435 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/.tmp/A/58de7bc062b04296bd7e6b4549827054 2024-11-27T12:25:39,756 DEBUG [RS_CLOSE_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/.tmp/B/8c581c4d6a20410181ab1d4ec70737d1 is 50, key is test_row_0/B:col10/1732710337029/Put/seqid=0 2024-11-27T12:25:39,760 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742148_1324 (size=9857) 2024-11-27T12:25:39,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-27T12:25:40,162 INFO [RS_CLOSE_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=435 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/.tmp/B/8c581c4d6a20410181ab1d4ec70737d1 2024-11-27T12:25:40,177 DEBUG [RS_CLOSE_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/.tmp/C/7a4f1610d80d4dacb56a321fecc88240 is 50, key is test_row_0/C:col10/1732710337029/Put/seqid=0 2024-11-27T12:25:40,182 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742149_1325 (size=9857) 2024-11-27T12:25:40,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-27T12:25:40,584 INFO [RS_CLOSE_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=435 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/.tmp/C/7a4f1610d80d4dacb56a321fecc88240 2024-11-27T12:25:40,595 DEBUG [RS_CLOSE_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/.tmp/A/58de7bc062b04296bd7e6b4549827054 as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/A/58de7bc062b04296bd7e6b4549827054 2024-11-27T12:25:40,602 INFO [RS_CLOSE_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/A/58de7bc062b04296bd7e6b4549827054, entries=100, sequenceid=435, filesize=9.6 K 2024-11-27T12:25:40,602 DEBUG [RS_CLOSE_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/.tmp/B/8c581c4d6a20410181ab1d4ec70737d1 as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/B/8c581c4d6a20410181ab1d4ec70737d1 2024-11-27T12:25:40,606 INFO [RS_CLOSE_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/B/8c581c4d6a20410181ab1d4ec70737d1, entries=100, sequenceid=435, filesize=9.6 K 2024-11-27T12:25:40,606 DEBUG [RS_CLOSE_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/.tmp/C/7a4f1610d80d4dacb56a321fecc88240 as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/C/7a4f1610d80d4dacb56a321fecc88240 2024-11-27T12:25:40,609 INFO [RS_CLOSE_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/C/7a4f1610d80d4dacb56a321fecc88240, entries=100, sequenceid=435, filesize=9.6 K 2024-11-27T12:25:40,610 INFO [RS_CLOSE_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for 3c3055e4b5609cf4625dfa6b13256b83 in 1281ms, sequenceid=435, compaction requested=true 2024-11-27T12:25:40,611 DEBUG [StoreCloser-TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/A/90f42d0014dc4971b27ced34e8b315c2, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/A/f4b453f643af48b5b9bb6e24dfb518f5, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/A/fc30337f9fa3475db6fbc578666ebc96, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/A/67e8e65a784045389118c6db13ad5655, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/A/010d9b792c054cd8b0aefc3ab09c2fbf, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/A/fb16592d6327417e81a6559188d7da16, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/A/bcc01fb81baa40eda70649ba9b89ad82, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/A/da7d72b097b54842acb90c5aa65fcbe6, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/A/e441396b293b4eb6bb49b597fe03adc0, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/A/032b5a7a90cb4aa2b41393bf6dd41061, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/A/82632257982c45a99f1b5235f8c8252e, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/A/0bed16d005ea457f849e554a21ba34d3, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/A/3862eb8dfd194caebcfa400f51b8b83a, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/A/bee2be8112594615bbc186c2d5260738, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/A/5b554eb6968b4e12b99cb76384381b1b, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/A/c493b74a85404fd5ac67218aceff4053, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/A/66312ea86dbb4acc8ef6006c989a0a61, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/A/e27aa87537994655bda6a2daae66f89c, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/A/cd91570099844461bf47f14d50dad1a6, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/A/cd36429e71aa4e649966af3901833a50, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/A/2b0c295e53fd41b7bc839062c8979128, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/A/fa238a787cd7411e81684fcf3a369372, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/A/597c37a28dc54656a5185dedeb61b32f, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/A/50083cd8a6c3484284932c1edcc7fe51, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/A/985de2a009aa43ed9c5e3211e09ddf1a, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/A/a8ebb76c9ed14e8bab981095e45c9463, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/A/868a6b712e4d47dd877151e825f2f056, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/A/26872716fb0d4608bfeeff0d231a1303] to archive 2024-11-27T12:25:40,612 DEBUG [StoreCloser-TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-27T12:25:40,613 DEBUG [StoreCloser-TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/A/90f42d0014dc4971b27ced34e8b315c2 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/A/90f42d0014dc4971b27ced34e8b315c2 2024-11-27T12:25:40,614 DEBUG [StoreCloser-TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/A/f4b453f643af48b5b9bb6e24dfb518f5 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/A/f4b453f643af48b5b9bb6e24dfb518f5 2024-11-27T12:25:40,615 DEBUG [StoreCloser-TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/A/fc30337f9fa3475db6fbc578666ebc96 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/A/fc30337f9fa3475db6fbc578666ebc96 2024-11-27T12:25:40,616 DEBUG [StoreCloser-TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/A/67e8e65a784045389118c6db13ad5655 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/A/67e8e65a784045389118c6db13ad5655 2024-11-27T12:25:40,617 DEBUG [StoreCloser-TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/A/010d9b792c054cd8b0aefc3ab09c2fbf to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/A/010d9b792c054cd8b0aefc3ab09c2fbf 2024-11-27T12:25:40,618 DEBUG [StoreCloser-TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/A/fb16592d6327417e81a6559188d7da16 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/A/fb16592d6327417e81a6559188d7da16 2024-11-27T12:25:40,619 DEBUG [StoreCloser-TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/A/bcc01fb81baa40eda70649ba9b89ad82 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/A/bcc01fb81baa40eda70649ba9b89ad82 2024-11-27T12:25:40,620 DEBUG [StoreCloser-TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/A/da7d72b097b54842acb90c5aa65fcbe6 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/A/da7d72b097b54842acb90c5aa65fcbe6 2024-11-27T12:25:40,620 DEBUG [StoreCloser-TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/A/e441396b293b4eb6bb49b597fe03adc0 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/A/e441396b293b4eb6bb49b597fe03adc0 2024-11-27T12:25:40,621 DEBUG [StoreCloser-TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/A/032b5a7a90cb4aa2b41393bf6dd41061 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/A/032b5a7a90cb4aa2b41393bf6dd41061 2024-11-27T12:25:40,622 DEBUG [StoreCloser-TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/A/82632257982c45a99f1b5235f8c8252e to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/A/82632257982c45a99f1b5235f8c8252e 2024-11-27T12:25:40,623 DEBUG [StoreCloser-TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/A/0bed16d005ea457f849e554a21ba34d3 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/A/0bed16d005ea457f849e554a21ba34d3 2024-11-27T12:25:40,624 DEBUG [StoreCloser-TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/A/3862eb8dfd194caebcfa400f51b8b83a to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/A/3862eb8dfd194caebcfa400f51b8b83a 2024-11-27T12:25:40,625 DEBUG [StoreCloser-TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/A/bee2be8112594615bbc186c2d5260738 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/A/bee2be8112594615bbc186c2d5260738 2024-11-27T12:25:40,626 DEBUG [StoreCloser-TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/A/5b554eb6968b4e12b99cb76384381b1b to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/A/5b554eb6968b4e12b99cb76384381b1b 2024-11-27T12:25:40,627 DEBUG [StoreCloser-TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/A/c493b74a85404fd5ac67218aceff4053 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/A/c493b74a85404fd5ac67218aceff4053 2024-11-27T12:25:40,629 DEBUG [StoreCloser-TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/A/66312ea86dbb4acc8ef6006c989a0a61 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/A/66312ea86dbb4acc8ef6006c989a0a61 2024-11-27T12:25:40,630 DEBUG [StoreCloser-TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/A/e27aa87537994655bda6a2daae66f89c to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/A/e27aa87537994655bda6a2daae66f89c 2024-11-27T12:25:40,631 DEBUG [StoreCloser-TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/A/cd91570099844461bf47f14d50dad1a6 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/A/cd91570099844461bf47f14d50dad1a6 2024-11-27T12:25:40,632 DEBUG [StoreCloser-TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/A/cd36429e71aa4e649966af3901833a50 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/A/cd36429e71aa4e649966af3901833a50 2024-11-27T12:25:40,633 DEBUG [StoreCloser-TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/A/2b0c295e53fd41b7bc839062c8979128 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/A/2b0c295e53fd41b7bc839062c8979128 2024-11-27T12:25:40,634 DEBUG [StoreCloser-TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/A/fa238a787cd7411e81684fcf3a369372 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/A/fa238a787cd7411e81684fcf3a369372 2024-11-27T12:25:40,635 DEBUG [StoreCloser-TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/A/597c37a28dc54656a5185dedeb61b32f to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/A/597c37a28dc54656a5185dedeb61b32f 2024-11-27T12:25:40,636 DEBUG [StoreCloser-TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/A/50083cd8a6c3484284932c1edcc7fe51 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/A/50083cd8a6c3484284932c1edcc7fe51 2024-11-27T12:25:40,637 DEBUG [StoreCloser-TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/A/985de2a009aa43ed9c5e3211e09ddf1a to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/A/985de2a009aa43ed9c5e3211e09ddf1a 2024-11-27T12:25:40,638 DEBUG [StoreCloser-TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/A/a8ebb76c9ed14e8bab981095e45c9463 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/A/a8ebb76c9ed14e8bab981095e45c9463 2024-11-27T12:25:40,640 DEBUG [StoreCloser-TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/A/868a6b712e4d47dd877151e825f2f056 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/A/868a6b712e4d47dd877151e825f2f056 2024-11-27T12:25:40,641 DEBUG [StoreCloser-TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/A/26872716fb0d4608bfeeff0d231a1303 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/A/26872716fb0d4608bfeeff0d231a1303 2024-11-27T12:25:40,642 DEBUG [StoreCloser-TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/B/afee21ebf75749a19fbc487182a483ea, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/B/d83f32d2f923428d85814b904e3f3638, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/B/cfdf1edf206c48839298e7083fdec231, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/B/ce147e8dccaa433d9649ec8cd42a7811, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/B/842a0a725343457b9486270b91a3f6c2, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/B/15cf97c1fa2a409b800d883ae16b1550, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/B/1126a40417a341cbb1db114f53cbb579, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/B/807bc563baf14cc485931c49c3666754, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/B/4cff65fc94e14a13a380bd4c81d4fb1a, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/B/e92a1c734b6e457797be0c0eecf7b586, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/B/ac573ba6a42d487185d30bd8e5881aef, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/B/b4b29f12a9c04d66b837557a085f8c0d, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/B/a8e383a16fb6432498bfc82cd4c3539e, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/B/e72b07f730c74167ab92afa7ab900b2e, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/B/6f72a4387d204bda9ce5046d5117891e, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/B/0da8d57788a549c7a9e737ad8269a4cc, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/B/1bb068ecb3104d7284298caeccde01d9, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/B/d5e23f3f4c8e4c8dbf926fe3d3a74c1a, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/B/23641de6b8364a86b271b4cabc02c466, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/B/3f25b0c093f8408dbe2df41b93c368f4, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/B/1c94e93a7bfc48f39c2649990c47802c, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/B/060bec5dc8e5470e9040ef4c0f0945a8, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/B/a275d028be1f4b62817936d556f6df3c, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/B/1d2ae34d54bf4236912e6107e1c4872b, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/B/4148add751e4419481b20abf006fe6b4, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/B/e1e1c93269d54e7aa4c20d4e0f39cf41, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/B/3fc3fe09dd474da1b979205b37affabc, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/B/1a0d050a9f944ab08dbc1ce632f0f47d] to archive 2024-11-27T12:25:40,643 DEBUG [StoreCloser-TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-27T12:25:40,644 DEBUG [StoreCloser-TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/B/afee21ebf75749a19fbc487182a483ea to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/B/afee21ebf75749a19fbc487182a483ea 2024-11-27T12:25:40,644 DEBUG [StoreCloser-TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/B/d83f32d2f923428d85814b904e3f3638 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/B/d83f32d2f923428d85814b904e3f3638 2024-11-27T12:25:40,645 DEBUG [StoreCloser-TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/B/cfdf1edf206c48839298e7083fdec231 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/B/cfdf1edf206c48839298e7083fdec231 2024-11-27T12:25:40,646 DEBUG [StoreCloser-TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/B/ce147e8dccaa433d9649ec8cd42a7811 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/B/ce147e8dccaa433d9649ec8cd42a7811 2024-11-27T12:25:40,647 DEBUG [StoreCloser-TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/B/842a0a725343457b9486270b91a3f6c2 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/B/842a0a725343457b9486270b91a3f6c2 2024-11-27T12:25:40,647 DEBUG [StoreCloser-TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/B/15cf97c1fa2a409b800d883ae16b1550 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/B/15cf97c1fa2a409b800d883ae16b1550 2024-11-27T12:25:40,648 DEBUG [StoreCloser-TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/B/1126a40417a341cbb1db114f53cbb579 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/B/1126a40417a341cbb1db114f53cbb579 2024-11-27T12:25:40,649 DEBUG [StoreCloser-TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/B/807bc563baf14cc485931c49c3666754 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/B/807bc563baf14cc485931c49c3666754 2024-11-27T12:25:40,650 DEBUG [StoreCloser-TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/B/4cff65fc94e14a13a380bd4c81d4fb1a to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/B/4cff65fc94e14a13a380bd4c81d4fb1a 2024-11-27T12:25:40,650 DEBUG [StoreCloser-TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/B/e92a1c734b6e457797be0c0eecf7b586 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/B/e92a1c734b6e457797be0c0eecf7b586 2024-11-27T12:25:40,651 DEBUG [StoreCloser-TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/B/ac573ba6a42d487185d30bd8e5881aef to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/B/ac573ba6a42d487185d30bd8e5881aef 2024-11-27T12:25:40,652 DEBUG [StoreCloser-TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/B/b4b29f12a9c04d66b837557a085f8c0d to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/B/b4b29f12a9c04d66b837557a085f8c0d 2024-11-27T12:25:40,652 DEBUG [StoreCloser-TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/B/a8e383a16fb6432498bfc82cd4c3539e to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/B/a8e383a16fb6432498bfc82cd4c3539e 2024-11-27T12:25:40,653 DEBUG [StoreCloser-TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/B/e72b07f730c74167ab92afa7ab900b2e to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/B/e72b07f730c74167ab92afa7ab900b2e 2024-11-27T12:25:40,654 DEBUG [StoreCloser-TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/B/6f72a4387d204bda9ce5046d5117891e to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/B/6f72a4387d204bda9ce5046d5117891e 2024-11-27T12:25:40,654 DEBUG [StoreCloser-TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/B/0da8d57788a549c7a9e737ad8269a4cc to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/B/0da8d57788a549c7a9e737ad8269a4cc 2024-11-27T12:25:40,655 DEBUG [StoreCloser-TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/B/1bb068ecb3104d7284298caeccde01d9 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/B/1bb068ecb3104d7284298caeccde01d9 2024-11-27T12:25:40,656 DEBUG [StoreCloser-TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/B/d5e23f3f4c8e4c8dbf926fe3d3a74c1a to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/B/d5e23f3f4c8e4c8dbf926fe3d3a74c1a 2024-11-27T12:25:40,657 DEBUG [StoreCloser-TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/B/23641de6b8364a86b271b4cabc02c466 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/B/23641de6b8364a86b271b4cabc02c466 2024-11-27T12:25:40,657 DEBUG [StoreCloser-TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/B/3f25b0c093f8408dbe2df41b93c368f4 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/B/3f25b0c093f8408dbe2df41b93c368f4 2024-11-27T12:25:40,658 DEBUG [StoreCloser-TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/B/1c94e93a7bfc48f39c2649990c47802c to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/B/1c94e93a7bfc48f39c2649990c47802c 2024-11-27T12:25:40,659 DEBUG [StoreCloser-TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/B/060bec5dc8e5470e9040ef4c0f0945a8 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/B/060bec5dc8e5470e9040ef4c0f0945a8 2024-11-27T12:25:40,660 DEBUG [StoreCloser-TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/B/a275d028be1f4b62817936d556f6df3c to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/B/a275d028be1f4b62817936d556f6df3c 2024-11-27T12:25:40,660 DEBUG [StoreCloser-TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/B/1d2ae34d54bf4236912e6107e1c4872b to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/B/1d2ae34d54bf4236912e6107e1c4872b 2024-11-27T12:25:40,661 DEBUG [StoreCloser-TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/B/4148add751e4419481b20abf006fe6b4 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/B/4148add751e4419481b20abf006fe6b4 2024-11-27T12:25:40,662 DEBUG [StoreCloser-TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/B/e1e1c93269d54e7aa4c20d4e0f39cf41 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/B/e1e1c93269d54e7aa4c20d4e0f39cf41 2024-11-27T12:25:40,662 DEBUG [StoreCloser-TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/B/3fc3fe09dd474da1b979205b37affabc to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/B/3fc3fe09dd474da1b979205b37affabc 2024-11-27T12:25:40,663 DEBUG [StoreCloser-TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/B/1a0d050a9f944ab08dbc1ce632f0f47d to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/B/1a0d050a9f944ab08dbc1ce632f0f47d 2024-11-27T12:25:40,664 DEBUG [StoreCloser-TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/C/8491cd57ae9148668e638d1315148c25, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/C/f68317e826e649659bd372d862b3e71b, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/C/06c9741688a7405884a3aa541a3226da, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/C/3b471d6df3f84234b24672a12e747fb6, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/C/6081cd35ef12471787e429634907855f, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/C/53394c9af01d4d5d9991e9c8d3aca180, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/C/ed0015bbefa8428fb4ccd28e6859ebe7, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/C/7e6e262460e24e09888db376683306fc, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/C/03aec8ffbd7244eda7166f699f133df3, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/C/4e8dd2abfd6c477694e660cf59e81595, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/C/e9848e3c8ca948aaa54f243144497bca, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/C/fdc5ebc404ba4ff39280b4e5ef60cf65, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/C/7e54f38d993b4e05b78daece4339348f, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/C/96775b63a8f94aeabe40c656fdd1b0fa, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/C/328a92f8c4d9445ab7c91fbfcc76c670, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/C/63b7c461bfc840daa82a35801d6ddeec, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/C/f4fe3f0978584b7da5fbf1fbb01c6cf4, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/C/85b8bd316f1b459ba34fe66f17a05c9a, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/C/8c565b953e4d4760b0e76b3c661fc4c8, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/C/21e3ef7a25ef4826b19cd77d44bf6736, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/C/82f3864ec8f342e7be9d13ceb7ebb340, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/C/e914b2695c21494c8b45eb11975f936f, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/C/0430b886bcd54740b8e10bc9b48f7997, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/C/9bf6b334ff694e30b39c574c6ad8d938, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/C/9c4220e9e36e4dd594ba60f8fca8ce02, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/C/03253546e9cd42fc9545f7232ceedbf2, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/C/592c6ed2afde456b8b71e2978e863715, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/C/c8e569b4d5034a31a22dcfd101e7b6e7] to archive 2024-11-27T12:25:40,665 DEBUG [StoreCloser-TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-27T12:25:40,666 DEBUG [StoreCloser-TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/C/8491cd57ae9148668e638d1315148c25 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/C/8491cd57ae9148668e638d1315148c25 2024-11-27T12:25:40,667 DEBUG [StoreCloser-TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/C/f68317e826e649659bd372d862b3e71b to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/C/f68317e826e649659bd372d862b3e71b 2024-11-27T12:25:40,668 DEBUG [StoreCloser-TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/C/06c9741688a7405884a3aa541a3226da to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/C/06c9741688a7405884a3aa541a3226da 2024-11-27T12:25:40,669 DEBUG [StoreCloser-TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/C/3b471d6df3f84234b24672a12e747fb6 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/C/3b471d6df3f84234b24672a12e747fb6 2024-11-27T12:25:40,670 DEBUG [StoreCloser-TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/C/6081cd35ef12471787e429634907855f to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/C/6081cd35ef12471787e429634907855f 2024-11-27T12:25:40,671 DEBUG [StoreCloser-TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/C/53394c9af01d4d5d9991e9c8d3aca180 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/C/53394c9af01d4d5d9991e9c8d3aca180 2024-11-27T12:25:40,672 DEBUG [StoreCloser-TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/C/ed0015bbefa8428fb4ccd28e6859ebe7 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/C/ed0015bbefa8428fb4ccd28e6859ebe7 2024-11-27T12:25:40,674 DEBUG [StoreCloser-TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/C/7e6e262460e24e09888db376683306fc to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/C/7e6e262460e24e09888db376683306fc 2024-11-27T12:25:40,674 DEBUG [StoreCloser-TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/C/03aec8ffbd7244eda7166f699f133df3 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/C/03aec8ffbd7244eda7166f699f133df3 2024-11-27T12:25:40,675 DEBUG [StoreCloser-TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/C/4e8dd2abfd6c477694e660cf59e81595 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/C/4e8dd2abfd6c477694e660cf59e81595 2024-11-27T12:25:40,677 DEBUG [StoreCloser-TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/C/e9848e3c8ca948aaa54f243144497bca to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/C/e9848e3c8ca948aaa54f243144497bca 2024-11-27T12:25:40,678 DEBUG [StoreCloser-TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/C/fdc5ebc404ba4ff39280b4e5ef60cf65 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/C/fdc5ebc404ba4ff39280b4e5ef60cf65 2024-11-27T12:25:40,679 DEBUG [StoreCloser-TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/C/7e54f38d993b4e05b78daece4339348f to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/C/7e54f38d993b4e05b78daece4339348f 2024-11-27T12:25:40,680 DEBUG [StoreCloser-TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/C/96775b63a8f94aeabe40c656fdd1b0fa to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/C/96775b63a8f94aeabe40c656fdd1b0fa 2024-11-27T12:25:40,681 DEBUG [StoreCloser-TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/C/328a92f8c4d9445ab7c91fbfcc76c670 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/C/328a92f8c4d9445ab7c91fbfcc76c670 2024-11-27T12:25:40,682 DEBUG [StoreCloser-TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/C/63b7c461bfc840daa82a35801d6ddeec to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/C/63b7c461bfc840daa82a35801d6ddeec 2024-11-27T12:25:40,684 DEBUG [StoreCloser-TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/C/f4fe3f0978584b7da5fbf1fbb01c6cf4 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/C/f4fe3f0978584b7da5fbf1fbb01c6cf4 2024-11-27T12:25:40,685 DEBUG [StoreCloser-TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/C/85b8bd316f1b459ba34fe66f17a05c9a to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/C/85b8bd316f1b459ba34fe66f17a05c9a 2024-11-27T12:25:40,686 DEBUG [StoreCloser-TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/C/8c565b953e4d4760b0e76b3c661fc4c8 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/C/8c565b953e4d4760b0e76b3c661fc4c8 2024-11-27T12:25:40,688 DEBUG [StoreCloser-TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/C/21e3ef7a25ef4826b19cd77d44bf6736 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/C/21e3ef7a25ef4826b19cd77d44bf6736 2024-11-27T12:25:40,690 DEBUG [StoreCloser-TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/C/82f3864ec8f342e7be9d13ceb7ebb340 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/C/82f3864ec8f342e7be9d13ceb7ebb340 2024-11-27T12:25:40,691 DEBUG [StoreCloser-TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/C/e914b2695c21494c8b45eb11975f936f to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/C/e914b2695c21494c8b45eb11975f936f 2024-11-27T12:25:40,693 DEBUG [StoreCloser-TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/C/0430b886bcd54740b8e10bc9b48f7997 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/C/0430b886bcd54740b8e10bc9b48f7997 2024-11-27T12:25:40,694 DEBUG [StoreCloser-TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/C/9bf6b334ff694e30b39c574c6ad8d938 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/C/9bf6b334ff694e30b39c574c6ad8d938 2024-11-27T12:25:40,696 DEBUG [StoreCloser-TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/C/9c4220e9e36e4dd594ba60f8fca8ce02 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/C/9c4220e9e36e4dd594ba60f8fca8ce02 2024-11-27T12:25:40,697 DEBUG [StoreCloser-TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/C/03253546e9cd42fc9545f7232ceedbf2 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/C/03253546e9cd42fc9545f7232ceedbf2 2024-11-27T12:25:40,699 DEBUG [StoreCloser-TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/C/592c6ed2afde456b8b71e2978e863715 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/C/592c6ed2afde456b8b71e2978e863715 2024-11-27T12:25:40,701 DEBUG [StoreCloser-TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/C/c8e569b4d5034a31a22dcfd101e7b6e7 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/C/c8e569b4d5034a31a22dcfd101e7b6e7 2024-11-27T12:25:40,707 DEBUG [RS_CLOSE_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/recovered.edits/438.seqid, newMaxSeqId=438, maxSeqId=1 2024-11-27T12:25:40,708 INFO [RS_CLOSE_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83. 2024-11-27T12:25:40,708 DEBUG [RS_CLOSE_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(1676): Region close journal for 3c3055e4b5609cf4625dfa6b13256b83: Waiting for close lock at 1732710339328Running coprocessor pre-close hooks at 1732710339329 (+1 ms)Disabling compacts and flushes for region at 1732710339329Disabling writes for close at 1732710339329Obtaining lock to block concurrent updates at 1732710339329Preparing flush snapshotting stores in 3c3055e4b5609cf4625dfa6b13256b83 at 1732710339329Finished memstore snapshotting TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=90720, getOffHeapSize=0, getCellsCount=750 at 1732710339330 (+1 ms)Flushing stores of TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83. at 1732710339331 (+1 ms)Flushing 3c3055e4b5609cf4625dfa6b13256b83/A: creating writer at 1732710339331Flushing 3c3055e4b5609cf4625dfa6b13256b83/A: appending metadata at 1732710339336 (+5 ms)Flushing 3c3055e4b5609cf4625dfa6b13256b83/A: closing flushed file at 1732710339336Flushing 3c3055e4b5609cf4625dfa6b13256b83/B: creating writer at 1732710339752 (+416 ms)Flushing 3c3055e4b5609cf4625dfa6b13256b83/B: appending metadata at 1732710339755 (+3 ms)Flushing 3c3055e4b5609cf4625dfa6b13256b83/B: closing flushed file at 1732710339755Flushing 3c3055e4b5609cf4625dfa6b13256b83/C: creating writer at 1732710340171 (+416 ms)Flushing 3c3055e4b5609cf4625dfa6b13256b83/C: appending metadata at 1732710340176 (+5 ms)Flushing 3c3055e4b5609cf4625dfa6b13256b83/C: closing flushed file at 1732710340176Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@723eba1a: reopening flushed file at 1732710340594 (+418 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2f42ea2a: reopening flushed file at 1732710340602 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@75d9d5d6: reopening flushed file at 1732710340606 (+4 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for 3c3055e4b5609cf4625dfa6b13256b83 in 1281ms, sequenceid=435, compaction requested=true at 1732710340610 (+4 ms)Writing region close event to WAL at 1732710340703 (+93 ms)Running coprocessor post-close hooks at 1732710340708 (+5 ms)Closed at 1732710340708 2024-11-27T12:25:40,711 INFO [RS_CLOSE_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] handler.UnassignRegionHandler(157): Closed 3c3055e4b5609cf4625dfa6b13256b83 2024-11-27T12:25:40,711 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=88 updating hbase:meta row=3c3055e4b5609cf4625dfa6b13256b83, regionState=CLOSED 2024-11-27T12:25:40,714 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=89, ppid=88, state=RUNNABLE, hasLock=false; CloseRegionProcedure 3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 because future has completed 2024-11-27T12:25:40,718 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=89, resume processing ppid=88 2024-11-27T12:25:40,718 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=89, ppid=88, state=SUCCESS, hasLock=false; CloseRegionProcedure 3c3055e4b5609cf4625dfa6b13256b83, server=1ba5af43fd99,36251,1732710241631 in 1.5400 sec 2024-11-27T12:25:40,721 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=88, resume processing ppid=87 2024-11-27T12:25:40,721 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=88, ppid=87, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=3c3055e4b5609cf4625dfa6b13256b83, UNASSIGN in 1.5460 sec 2024-11-27T12:25:40,724 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=87, resume processing ppid=86 2024-11-27T12:25:40,725 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=87, ppid=86, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.5520 sec 2024-11-27T12:25:40,726 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732710340726"}]},"ts":"1732710340726"} 2024-11-27T12:25:40,728 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-27T12:25:40,728 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-27T12:25:40,730 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=86, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.5670 sec 2024-11-27T12:25:41,197 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-27T12:25:41,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-27T12:25:41,309 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-27T12:25:41,309 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36215 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-11-27T12:25:41,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36215 {}] procedure2.ProcedureExecutor(1139): Stored pid=90, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-27T12:25:41,311 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=90, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-27T12:25:41,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=90 2024-11-27T12:25:41,311 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=90, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-27T12:25:41,314 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83 2024-11-27T12:25:41,316 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/A, FileablePath, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/B, FileablePath, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/C, FileablePath, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/recovered.edits] 2024-11-27T12:25:41,319 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/A/58de7bc062b04296bd7e6b4549827054 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/A/58de7bc062b04296bd7e6b4549827054 2024-11-27T12:25:41,321 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/A/aee3313394af49f18f941aa9790176fb to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/A/aee3313394af49f18f941aa9790176fb 2024-11-27T12:25:41,323 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/A/dd0427338e7e4e6a876beb14e46d5792 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/A/dd0427338e7e4e6a876beb14e46d5792 2024-11-27T12:25:41,326 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/B/7604d4059638431683177b07f606ee25 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/B/7604d4059638431683177b07f606ee25 2024-11-27T12:25:41,328 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/B/8c581c4d6a20410181ab1d4ec70737d1 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/B/8c581c4d6a20410181ab1d4ec70737d1 2024-11-27T12:25:41,330 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/B/f1a18069e2b746739d48b00cd08c1906 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/B/f1a18069e2b746739d48b00cd08c1906 2024-11-27T12:25:41,333 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/C/55df93d09b9e464487348302ea9f3e59 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/C/55df93d09b9e464487348302ea9f3e59 2024-11-27T12:25:41,335 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/C/7a4f1610d80d4dacb56a321fecc88240 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/C/7a4f1610d80d4dacb56a321fecc88240 2024-11-27T12:25:41,338 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/C/8bd691e8afd7491eaa572d0edf1a85ab to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/C/8bd691e8afd7491eaa572d0edf1a85ab 2024-11-27T12:25:41,342 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/recovered.edits/438.seqid to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83/recovered.edits/438.seqid 2024-11-27T12:25:41,343 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3c3055e4b5609cf4625dfa6b13256b83 2024-11-27T12:25:41,343 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-27T12:25:41,347 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=90, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-27T12:25:41,351 WARN [PEWorker-2 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-27T12:25:41,354 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-27T12:25:41,356 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=90, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-27T12:25:41,356 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-27T12:25:41,356 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732710341356"}]},"ts":"9223372036854775807"} 2024-11-27T12:25:41,358 INFO [PEWorker-2 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-27T12:25:41,358 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 3c3055e4b5609cf4625dfa6b13256b83, NAME => 'TestAcidGuarantees,,1732710314692.3c3055e4b5609cf4625dfa6b13256b83.', STARTKEY => '', ENDKEY => ''}] 2024-11-27T12:25:41,359 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-27T12:25:41,359 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732710341359"}]},"ts":"9223372036854775807"} 2024-11-27T12:25:41,361 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-27T12:25:41,362 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(133): Finished pid=90, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-27T12:25:41,364 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=90, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 53 msec 2024-11-27T12:25:41,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=90 2024-11-27T12:25:41,420 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-27T12:25:41,420 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-27T12:25:41,435 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testGetAtomicity Thread=240 (was 238) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1127887625_22 at /127.0.0.1:37466 [Waiting for operation #389] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1180310523_22 at /127.0.0.1:37812 [Waiting for operation #195] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=453 (was 449) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=457 (was 416) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=8615 (was 9349) 2024-11-27T12:25:41,444 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMobScanAtomicity Thread=240, OpenFileDescriptor=453, MaxFileDescriptor=1048576, SystemLoadAverage=457, ProcessCount=11, AvailableMemoryMB=8615 2024-11-27T12:25:41,445 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36215 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-27T12:25:41,446 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36215 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-27T12:25:41,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36215 {}] procedure2.ProcedureExecutor(1139): Stored pid=91, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-27T12:25:41,448 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=91, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-27T12:25:41,448 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:25:41,448 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36215 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 91 2024-11-27T12:25:41,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-27T12:25:41,449 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=91, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-27T12:25:41,454 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742150_1326 (size=959) 2024-11-27T12:25:41,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-27T12:25:41,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-27T12:25:41,856 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 817ba5bcb98df62e5ed309caecfc1068, NAME => 'TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c 2024-11-27T12:25:41,861 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742151_1327 (size=53) 2024-11-27T12:25:41,862 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T12:25:41,862 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 817ba5bcb98df62e5ed309caecfc1068, disabling compactions & flushes 2024-11-27T12:25:41,862 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068. 2024-11-27T12:25:41,862 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068. 2024-11-27T12:25:41,862 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068. after waiting 0 ms 2024-11-27T12:25:41,862 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068. 2024-11-27T12:25:41,862 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068. 2024-11-27T12:25:41,862 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 817ba5bcb98df62e5ed309caecfc1068: Waiting for close lock at 1732710341862Disabling compacts and flushes for region at 1732710341862Disabling writes for close at 1732710341862Writing region close event to WAL at 1732710341862Closed at 1732710341862 2024-11-27T12:25:41,864 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=91, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-27T12:25:41,864 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732710341864"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732710341864"}]},"ts":"1732710341864"} 2024-11-27T12:25:41,865 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-27T12:25:41,866 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=91, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-27T12:25:41,866 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732710341866"}]},"ts":"1732710341866"} 2024-11-27T12:25:41,868 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-27T12:25:41,868 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=92, ppid=91, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=817ba5bcb98df62e5ed309caecfc1068, ASSIGN}] 2024-11-27T12:25:41,869 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=92, ppid=91, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=817ba5bcb98df62e5ed309caecfc1068, ASSIGN 2024-11-27T12:25:41,869 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=92, ppid=91, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=817ba5bcb98df62e5ed309caecfc1068, ASSIGN; state=OFFLINE, location=1ba5af43fd99,36251,1732710241631; forceNewPlan=false, retain=false 2024-11-27T12:25:42,020 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=92 updating hbase:meta row=817ba5bcb98df62e5ed309caecfc1068, regionState=OPENING, regionLocation=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:42,022 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=92, ppid=91, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=817ba5bcb98df62e5ed309caecfc1068, ASSIGN because future has completed 2024-11-27T12:25:42,023 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=93, ppid=92, state=RUNNABLE, hasLock=false; OpenRegionProcedure 817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631}] 2024-11-27T12:25:42,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-27T12:25:42,177 INFO [RS_OPEN_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068. 2024-11-27T12:25:42,178 DEBUG [RS_OPEN_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(7752): Opening region: {ENCODED => 817ba5bcb98df62e5ed309caecfc1068, NAME => 'TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068.', STARTKEY => '', ENDKEY => ''} 2024-11-27T12:25:42,178 DEBUG [RS_OPEN_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 817ba5bcb98df62e5ed309caecfc1068 2024-11-27T12:25:42,178 DEBUG [RS_OPEN_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T12:25:42,178 DEBUG [RS_OPEN_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(7794): checking encryption for 817ba5bcb98df62e5ed309caecfc1068 2024-11-27T12:25:42,178 DEBUG [RS_OPEN_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(7797): checking classloading for 817ba5bcb98df62e5ed309caecfc1068 2024-11-27T12:25:42,180 INFO [StoreOpener-817ba5bcb98df62e5ed309caecfc1068-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 817ba5bcb98df62e5ed309caecfc1068 2024-11-27T12:25:42,181 INFO [StoreOpener-817ba5bcb98df62e5ed309caecfc1068-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 817ba5bcb98df62e5ed309caecfc1068 columnFamilyName A 2024-11-27T12:25:42,181 DEBUG [StoreOpener-817ba5bcb98df62e5ed309caecfc1068-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:25:42,182 INFO [StoreOpener-817ba5bcb98df62e5ed309caecfc1068-1 {}] regionserver.HStore(327): Store=817ba5bcb98df62e5ed309caecfc1068/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T12:25:42,182 INFO [StoreOpener-817ba5bcb98df62e5ed309caecfc1068-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 817ba5bcb98df62e5ed309caecfc1068 2024-11-27T12:25:42,183 INFO [StoreOpener-817ba5bcb98df62e5ed309caecfc1068-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 817ba5bcb98df62e5ed309caecfc1068 columnFamilyName B 2024-11-27T12:25:42,183 DEBUG [StoreOpener-817ba5bcb98df62e5ed309caecfc1068-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:25:42,183 INFO [StoreOpener-817ba5bcb98df62e5ed309caecfc1068-1 {}] regionserver.HStore(327): Store=817ba5bcb98df62e5ed309caecfc1068/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T12:25:42,183 INFO [StoreOpener-817ba5bcb98df62e5ed309caecfc1068-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 817ba5bcb98df62e5ed309caecfc1068 2024-11-27T12:25:42,184 INFO [StoreOpener-817ba5bcb98df62e5ed309caecfc1068-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 817ba5bcb98df62e5ed309caecfc1068 columnFamilyName C 2024-11-27T12:25:42,184 DEBUG [StoreOpener-817ba5bcb98df62e5ed309caecfc1068-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:25:42,185 INFO [StoreOpener-817ba5bcb98df62e5ed309caecfc1068-1 {}] regionserver.HStore(327): Store=817ba5bcb98df62e5ed309caecfc1068/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T12:25:42,185 DEBUG [RS_OPEN_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(1038): replaying wal for 817ba5bcb98df62e5ed309caecfc1068 2024-11-27T12:25:42,186 DEBUG [RS_OPEN_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068 2024-11-27T12:25:42,186 DEBUG [RS_OPEN_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068 2024-11-27T12:25:42,187 DEBUG [RS_OPEN_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(1048): stopping wal replay for 817ba5bcb98df62e5ed309caecfc1068 2024-11-27T12:25:42,187 DEBUG [RS_OPEN_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(1060): Cleaning up temporary data for 817ba5bcb98df62e5ed309caecfc1068 2024-11-27T12:25:42,187 DEBUG [RS_OPEN_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-27T12:25:42,188 DEBUG [RS_OPEN_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(1093): writing seq id for 817ba5bcb98df62e5ed309caecfc1068 2024-11-27T12:25:42,189 DEBUG [RS_OPEN_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-27T12:25:42,190 INFO [RS_OPEN_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(1114): Opened 817ba5bcb98df62e5ed309caecfc1068; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72867514, jitterRate=0.0858105719089508}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-27T12:25:42,190 DEBUG [RS_OPEN_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 817ba5bcb98df62e5ed309caecfc1068 2024-11-27T12:25:42,191 DEBUG [RS_OPEN_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(1006): Region open journal for 817ba5bcb98df62e5ed309caecfc1068: Running coprocessor pre-open hook at 1732710342179Writing region info on filesystem at 1732710342179Initializing all the Stores at 1732710342179Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732710342179Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732710342179Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732710342180 (+1 ms)Cleaning up temporary data from old regions at 1732710342187 (+7 ms)Running coprocessor post-open hooks at 1732710342190 (+3 ms)Region opened successfully at 1732710342191 (+1 ms) 2024-11-27T12:25:42,192 INFO [RS_OPEN_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., pid=93, masterSystemTime=1732710342174 2024-11-27T12:25:42,194 DEBUG [RS_OPEN_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068. 2024-11-27T12:25:42,194 INFO [RS_OPEN_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068. 2024-11-27T12:25:42,195 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=92 updating hbase:meta row=817ba5bcb98df62e5ed309caecfc1068, regionState=OPEN, openSeqNum=2, regionLocation=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:42,196 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=93, ppid=92, state=RUNNABLE, hasLock=false; OpenRegionProcedure 817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 because future has completed 2024-11-27T12:25:42,200 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=93, resume processing ppid=92 2024-11-27T12:25:42,200 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=93, ppid=92, state=SUCCESS, hasLock=false; OpenRegionProcedure 817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 in 176 msec 2024-11-27T12:25:42,202 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=92, resume processing ppid=91 2024-11-27T12:25:42,202 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=92, ppid=91, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=817ba5bcb98df62e5ed309caecfc1068, ASSIGN in 332 msec 2024-11-27T12:25:42,203 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=91, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-27T12:25:42,204 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732710342203"}]},"ts":"1732710342203"} 2024-11-27T12:25:42,205 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-27T12:25:42,206 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=91, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-27T12:25:42,208 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=91, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 760 msec 2024-11-27T12:25:42,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-27T12:25:42,589 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-27T12:25:42,590 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@523828c8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T12:25:42,590 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1ba5af43fd99,36215,-1 for getting cluster id 2024-11-27T12:25:42,590 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T12:25:42,591 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '200978c2-2c41-4f93-b0b5-127ea1024f37' 2024-11-27T12:25:42,591 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T12:25:42,591 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "200978c2-2c41-4f93-b0b5-127ea1024f37" 2024-11-27T12:25:42,592 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1a005ecb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T12:25:42,592 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1ba5af43fd99,36215,-1] 2024-11-27T12:25:42,592 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T12:25:42,592 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T12:25:42,593 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46150, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T12:25:42,594 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@433f4ca6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T12:25:42,594 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T12:25:42,595 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1ba5af43fd99,36251,1732710241631, seqNum=-1] 2024-11-27T12:25:42,595 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T12:25:42,596 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60062, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T12:25:42,597 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-27T12:25:42,598 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is 1ba5af43fd99,36215,1732710240860 2024-11-27T12:25:42,598 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@280d2d26 2024-11-27T12:25:42,598 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-27T12:25:42,599 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46152, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-27T12:25:42,601 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36215 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-27T12:25:42,601 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36215 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.3 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-27T12:25:42,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36215 {}] procedure2.ProcedureExecutor(1139): Stored pid=94, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-27T12:25:42,613 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742152_1328 (size=995) 2024-11-27T12:25:42,615 DEBUG [PEWorker-2 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.959 2024-11-27T12:25:42,615 INFO [PEWorker-2 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.995 2024-11-27T12:25:42,618 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=95, ppid=94, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-27T12:25:42,620 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=96, ppid=95, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=817ba5bcb98df62e5ed309caecfc1068, REOPEN/MOVE}] 2024-11-27T12:25:42,622 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=96, ppid=95, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=817ba5bcb98df62e5ed309caecfc1068, REOPEN/MOVE 2024-11-27T12:25:42,623 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=96 updating hbase:meta row=817ba5bcb98df62e5ed309caecfc1068, regionState=CLOSING, regionLocation=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:42,624 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=96, ppid=95, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=817ba5bcb98df62e5ed309caecfc1068, REOPEN/MOVE because future has completed 2024-11-27T12:25:42,624 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-27T12:25:42,625 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=97, ppid=96, state=RUNNABLE, hasLock=false; CloseRegionProcedure 817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631}] 2024-11-27T12:25:42,777 INFO [RS_CLOSE_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] handler.UnassignRegionHandler(122): Close 817ba5bcb98df62e5ed309caecfc1068 2024-11-27T12:25:42,777 DEBUG [RS_CLOSE_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-27T12:25:42,777 DEBUG [RS_CLOSE_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.HRegion(1722): Closing 817ba5bcb98df62e5ed309caecfc1068, disabling compactions & flushes 2024-11-27T12:25:42,777 INFO [RS_CLOSE_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068. 2024-11-27T12:25:42,777 DEBUG [RS_CLOSE_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068. 2024-11-27T12:25:42,777 DEBUG [RS_CLOSE_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068. after waiting 0 ms 2024-11-27T12:25:42,777 DEBUG [RS_CLOSE_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068. 2024-11-27T12:25:42,780 DEBUG [RS_CLOSE_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-27T12:25:42,781 INFO [RS_CLOSE_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068. 2024-11-27T12:25:42,781 DEBUG [RS_CLOSE_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.HRegion(1676): Region close journal for 817ba5bcb98df62e5ed309caecfc1068: Waiting for close lock at 1732710342777Running coprocessor pre-close hooks at 1732710342777Disabling compacts and flushes for region at 1732710342777Disabling writes for close at 1732710342777Writing region close event to WAL at 1732710342777Running coprocessor post-close hooks at 1732710342781 (+4 ms)Closed at 1732710342781 2024-11-27T12:25:42,781 WARN [RS_CLOSE_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.HRegionServer(3299): Not adding moved region record: 817ba5bcb98df62e5ed309caecfc1068 to self. 2024-11-27T12:25:42,784 INFO [RS_CLOSE_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] handler.UnassignRegionHandler(157): Closed 817ba5bcb98df62e5ed309caecfc1068 2024-11-27T12:25:42,784 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=96 updating hbase:meta row=817ba5bcb98df62e5ed309caecfc1068, regionState=CLOSED 2024-11-27T12:25:42,786 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=97, ppid=96, state=RUNNABLE, hasLock=false; CloseRegionProcedure 817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 because future has completed 2024-11-27T12:25:42,789 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=97, resume processing ppid=96 2024-11-27T12:25:42,789 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=97, ppid=96, state=SUCCESS, hasLock=false; CloseRegionProcedure 817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 in 162 msec 2024-11-27T12:25:42,790 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=96, ppid=95, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=817ba5bcb98df62e5ed309caecfc1068, REOPEN/MOVE; state=CLOSED, location=1ba5af43fd99,36251,1732710241631; forceNewPlan=false, retain=true 2024-11-27T12:25:42,941 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=96 updating hbase:meta row=817ba5bcb98df62e5ed309caecfc1068, regionState=OPENING, regionLocation=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:42,942 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=96, ppid=95, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=817ba5bcb98df62e5ed309caecfc1068, REOPEN/MOVE because future has completed 2024-11-27T12:25:42,943 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=98, ppid=96, state=RUNNABLE, hasLock=false; OpenRegionProcedure 817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631}] 2024-11-27T12:25:43,097 INFO [RS_OPEN_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068. 2024-11-27T12:25:43,097 DEBUG [RS_OPEN_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(7752): Opening region: {ENCODED => 817ba5bcb98df62e5ed309caecfc1068, NAME => 'TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068.', STARTKEY => '', ENDKEY => ''} 2024-11-27T12:25:43,098 DEBUG [RS_OPEN_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 817ba5bcb98df62e5ed309caecfc1068 2024-11-27T12:25:43,098 DEBUG [RS_OPEN_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T12:25:43,098 DEBUG [RS_OPEN_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(7794): checking encryption for 817ba5bcb98df62e5ed309caecfc1068 2024-11-27T12:25:43,098 DEBUG [RS_OPEN_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(7797): checking classloading for 817ba5bcb98df62e5ed309caecfc1068 2024-11-27T12:25:43,099 INFO [StoreOpener-817ba5bcb98df62e5ed309caecfc1068-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 817ba5bcb98df62e5ed309caecfc1068 2024-11-27T12:25:43,100 INFO [StoreOpener-817ba5bcb98df62e5ed309caecfc1068-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 817ba5bcb98df62e5ed309caecfc1068 columnFamilyName A 2024-11-27T12:25:43,101 DEBUG [StoreOpener-817ba5bcb98df62e5ed309caecfc1068-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:25:43,102 INFO [StoreOpener-817ba5bcb98df62e5ed309caecfc1068-1 {}] regionserver.HStore(327): Store=817ba5bcb98df62e5ed309caecfc1068/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T12:25:43,102 INFO [StoreOpener-817ba5bcb98df62e5ed309caecfc1068-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 817ba5bcb98df62e5ed309caecfc1068 2024-11-27T12:25:43,103 INFO [StoreOpener-817ba5bcb98df62e5ed309caecfc1068-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 817ba5bcb98df62e5ed309caecfc1068 columnFamilyName B 2024-11-27T12:25:43,103 DEBUG [StoreOpener-817ba5bcb98df62e5ed309caecfc1068-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:25:43,104 INFO [StoreOpener-817ba5bcb98df62e5ed309caecfc1068-1 {}] regionserver.HStore(327): Store=817ba5bcb98df62e5ed309caecfc1068/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T12:25:43,104 INFO [StoreOpener-817ba5bcb98df62e5ed309caecfc1068-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 817ba5bcb98df62e5ed309caecfc1068 2024-11-27T12:25:43,105 INFO [StoreOpener-817ba5bcb98df62e5ed309caecfc1068-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 817ba5bcb98df62e5ed309caecfc1068 columnFamilyName C 2024-11-27T12:25:43,105 DEBUG [StoreOpener-817ba5bcb98df62e5ed309caecfc1068-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:25:43,105 INFO [StoreOpener-817ba5bcb98df62e5ed309caecfc1068-1 {}] regionserver.HStore(327): Store=817ba5bcb98df62e5ed309caecfc1068/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T12:25:43,106 DEBUG [RS_OPEN_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(1038): replaying wal for 817ba5bcb98df62e5ed309caecfc1068 2024-11-27T12:25:43,106 DEBUG [RS_OPEN_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068 2024-11-27T12:25:43,108 DEBUG [RS_OPEN_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068 2024-11-27T12:25:43,109 DEBUG [RS_OPEN_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(1048): stopping wal replay for 817ba5bcb98df62e5ed309caecfc1068 2024-11-27T12:25:43,109 DEBUG [RS_OPEN_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(1060): Cleaning up temporary data for 817ba5bcb98df62e5ed309caecfc1068 2024-11-27T12:25:43,109 DEBUG [RS_OPEN_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-27T12:25:43,111 DEBUG [RS_OPEN_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(1093): writing seq id for 817ba5bcb98df62e5ed309caecfc1068 2024-11-27T12:25:43,112 INFO [RS_OPEN_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(1114): Opened 817ba5bcb98df62e5ed309caecfc1068; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61693495, jitterRate=-0.08069528639316559}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-27T12:25:43,112 DEBUG [RS_OPEN_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 817ba5bcb98df62e5ed309caecfc1068 2024-11-27T12:25:43,113 DEBUG [RS_OPEN_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(1006): Region open journal for 817ba5bcb98df62e5ed309caecfc1068: Running coprocessor pre-open hook at 1732710343098Writing region info on filesystem at 1732710343098Initializing all the Stores at 1732710343099 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732710343099Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732710343099Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732710343099Cleaning up temporary data from old regions at 1732710343109 (+10 ms)Running coprocessor post-open hooks at 1732710343112 (+3 ms)Region opened successfully at 1732710343113 (+1 ms) 2024-11-27T12:25:43,114 INFO [RS_OPEN_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., pid=98, masterSystemTime=1732710343094 2024-11-27T12:25:43,116 DEBUG [RS_OPEN_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068. 2024-11-27T12:25:43,116 INFO [RS_OPEN_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068. 2024-11-27T12:25:43,116 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=96 updating hbase:meta row=817ba5bcb98df62e5ed309caecfc1068, regionState=OPEN, openSeqNum=5, regionLocation=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:43,118 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=98, ppid=96, state=RUNNABLE, hasLock=false; OpenRegionProcedure 817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 because future has completed 2024-11-27T12:25:43,121 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=98, resume processing ppid=96 2024-11-27T12:25:43,121 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=98, ppid=96, state=SUCCESS, hasLock=false; OpenRegionProcedure 817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 in 176 msec 2024-11-27T12:25:43,123 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=96, resume processing ppid=95 2024-11-27T12:25:43,123 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=96, ppid=95, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=817ba5bcb98df62e5ed309caecfc1068, REOPEN/MOVE in 501 msec 2024-11-27T12:25:43,125 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=95, resume processing ppid=94 2024-11-27T12:25:43,125 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=95, ppid=94, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 505 msec 2024-11-27T12:25:43,128 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=94, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 524 msec 2024-11-27T12:25:43,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=94 2024-11-27T12:25:43,129 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-27T12:25:43,130 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@150b6d4f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T12:25:43,130 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1ba5af43fd99,36215,-1 for getting cluster id 2024-11-27T12:25:43,130 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T12:25:43,131 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '200978c2-2c41-4f93-b0b5-127ea1024f37' 2024-11-27T12:25:43,131 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T12:25:43,131 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "200978c2-2c41-4f93-b0b5-127ea1024f37" 2024-11-27T12:25:43,131 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ec0772b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T12:25:43,131 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1ba5af43fd99,36215,-1] 2024-11-27T12:25:43,132 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T12:25:43,132 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T12:25:43,133 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46160, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T12:25:43,133 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5066351c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T12:25:43,134 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5ec79e6b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T12:25:43,134 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1ba5af43fd99,36215,-1 for getting cluster id 2024-11-27T12:25:43,134 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T12:25:43,135 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '200978c2-2c41-4f93-b0b5-127ea1024f37' 2024-11-27T12:25:43,135 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T12:25:43,136 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "200978c2-2c41-4f93-b0b5-127ea1024f37" 2024-11-27T12:25:43,136 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7b4a0a90, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T12:25:43,136 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1ba5af43fd99,36215,-1] 2024-11-27T12:25:43,136 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T12:25:43,136 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T12:25:43,137 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46166, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T12:25:43,138 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@69887c23, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T12:25:43,138 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6d70a5ba, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T12:25:43,138 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1ba5af43fd99,36215,-1 for getting cluster id 2024-11-27T12:25:43,139 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T12:25:43,139 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '200978c2-2c41-4f93-b0b5-127ea1024f37' 2024-11-27T12:25:43,140 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T12:25:43,140 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "200978c2-2c41-4f93-b0b5-127ea1024f37" 2024-11-27T12:25:43,140 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7cb8e4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T12:25:43,140 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1ba5af43fd99,36215,-1] 2024-11-27T12:25:43,140 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T12:25:43,140 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T12:25:43,141 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46194, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T12:25:43,142 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@519f2904, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T12:25:43,143 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@436a7f64, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T12:25:43,143 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1ba5af43fd99,36215,-1 for getting cluster id 2024-11-27T12:25:43,143 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T12:25:43,144 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '200978c2-2c41-4f93-b0b5-127ea1024f37' 2024-11-27T12:25:43,144 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T12:25:43,144 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "200978c2-2c41-4f93-b0b5-127ea1024f37" 2024-11-27T12:25:43,144 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5a1abf9f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T12:25:43,144 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1ba5af43fd99,36215,-1] 2024-11-27T12:25:43,145 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T12:25:43,145 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T12:25:43,146 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46212, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T12:25:43,146 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@323b88d6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T12:25:43,147 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@40d8da2d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T12:25:43,147 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1ba5af43fd99,36215,-1 for getting cluster id 2024-11-27T12:25:43,147 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T12:25:43,148 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '200978c2-2c41-4f93-b0b5-127ea1024f37' 2024-11-27T12:25:43,148 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T12:25:43,148 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "200978c2-2c41-4f93-b0b5-127ea1024f37" 2024-11-27T12:25:43,148 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@736c3a01, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T12:25:43,148 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1ba5af43fd99,36215,-1] 2024-11-27T12:25:43,149 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T12:25:43,149 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T12:25:43,150 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46230, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T12:25:43,150 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1af2f16f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T12:25:43,151 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1c61df40, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T12:25:43,151 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1ba5af43fd99,36215,-1 for getting cluster id 2024-11-27T12:25:43,152 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T12:25:43,153 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '200978c2-2c41-4f93-b0b5-127ea1024f37' 2024-11-27T12:25:43,153 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T12:25:43,153 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "200978c2-2c41-4f93-b0b5-127ea1024f37" 2024-11-27T12:25:43,153 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@ffc8ae1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T12:25:43,153 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1ba5af43fd99,36215,-1] 2024-11-27T12:25:43,153 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T12:25:43,154 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T12:25:43,154 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46248, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T12:25:43,155 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@21a74bf2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T12:25:43,156 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32435f2d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T12:25:43,156 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1ba5af43fd99,36215,-1 for getting cluster id 2024-11-27T12:25:43,156 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T12:25:43,157 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '200978c2-2c41-4f93-b0b5-127ea1024f37' 2024-11-27T12:25:43,157 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T12:25:43,157 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "200978c2-2c41-4f93-b0b5-127ea1024f37" 2024-11-27T12:25:43,157 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@798c10f9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T12:25:43,157 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1ba5af43fd99,36215,-1] 2024-11-27T12:25:43,158 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T12:25:43,158 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T12:25:43,159 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46264, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T12:25:43,159 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@35425861, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T12:25:43,160 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@15f1ed2b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T12:25:43,160 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1ba5af43fd99,36215,-1 for getting cluster id 2024-11-27T12:25:43,160 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T12:25:43,161 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '200978c2-2c41-4f93-b0b5-127ea1024f37' 2024-11-27T12:25:43,162 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T12:25:43,162 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "200978c2-2c41-4f93-b0b5-127ea1024f37" 2024-11-27T12:25:43,162 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a8594e9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T12:25:43,162 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1ba5af43fd99,36215,-1] 2024-11-27T12:25:43,162 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T12:25:43,163 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T12:25:43,163 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46278, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T12:25:43,164 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c2b0974, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T12:25:43,165 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b9f15fa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T12:25:43,165 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1ba5af43fd99,36215,-1 for getting cluster id 2024-11-27T12:25:43,165 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T12:25:43,166 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '200978c2-2c41-4f93-b0b5-127ea1024f37' 2024-11-27T12:25:43,166 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T12:25:43,167 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "200978c2-2c41-4f93-b0b5-127ea1024f37" 2024-11-27T12:25:43,167 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@57d33abf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T12:25:43,167 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1ba5af43fd99,36215,-1] 2024-11-27T12:25:43,167 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T12:25:43,167 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T12:25:43,168 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46284, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T12:25:43,169 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7ba69008, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T12:25:43,170 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@763c79a5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T12:25:43,170 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1ba5af43fd99,36215,-1 for getting cluster id 2024-11-27T12:25:43,170 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T12:25:43,171 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '200978c2-2c41-4f93-b0b5-127ea1024f37' 2024-11-27T12:25:43,171 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T12:25:43,171 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "200978c2-2c41-4f93-b0b5-127ea1024f37" 2024-11-27T12:25:43,172 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@720bcc22, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T12:25:43,172 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1ba5af43fd99,36215,-1] 2024-11-27T12:25:43,172 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T12:25:43,172 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T12:25:43,173 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46304, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T12:25:43,174 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1cc1dad2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T12:25:43,175 DEBUG [Thread-1515 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T12:25:43,176 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1ba5af43fd99,36251,1732710241631, seqNum=-1] 2024-11-27T12:25:43,176 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T12:25:43,176 DEBUG [Thread-1502 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T12:25:43,177 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60068, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T12:25:43,177 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1ba5af43fd99,36251,1732710241631, seqNum=-1] 2024-11-27T12:25:43,178 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T12:25:43,179 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60076, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T12:25:43,179 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5] 2024-11-27T12:25:43,181 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5] 2024-11-27T12:25:43,188 DEBUG [Thread-1513 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T12:25:43,189 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1ba5af43fd99,36251,1732710241631, seqNum=-1] 2024-11-27T12:25:43,190 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T12:25:43,191 DEBUG [Thread-1519 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T12:25:43,192 DEBUG [Thread-1519 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1ba5af43fd99,36251,1732710241631, seqNum=-1] 2024-11-27T12:25:43,192 DEBUG [Thread-1519 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T12:25:43,193 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60082, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T12:25:43,194 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60096, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T12:25:43,194 DEBUG [Thread-1506 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T12:25:43,207 DEBUG [Thread-1517 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T12:25:43,207 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-27T12:25:43,208 DEBUG [Thread-1510 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T12:25:43,208 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1ba5af43fd99,36251,1732710241631, seqNum=-1] 2024-11-27T12:25:43,208 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1ba5af43fd99,36251,1732710241631, seqNum=-1] 2024-11-27T12:25:43,209 DEBUG [Thread-1521 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T12:25:43,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] procedure2.ProcedureExecutor(1139): Stored pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees 2024-11-27T12:25:43,209 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5] 2024-11-27T12:25:43,209 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T12:25:43,209 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T12:25:43,210 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60108, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T12:25:43,210 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60110, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T12:25:43,210 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5] 2024-11-27T12:25:43,211 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T12:25:43,211 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1ba5af43fd99,36251,1732710241631, seqNum=-1] 2024-11-27T12:25:43,211 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T12:25:43,212 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T12:25:43,212 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=100, ppid=99, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T12:25:43,212 DEBUG [Thread-1521 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1ba5af43fd99,36251,1732710241631, seqNum=-1] 2024-11-27T12:25:43,212 DEBUG [Thread-1521 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T12:25:43,213 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60134, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T12:25:43,213 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60148, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T12:25:43,215 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5] 2024-11-27T12:25:43,216 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5] 2024-11-27T12:25:43,216 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5] 2024-11-27T12:25:43,219 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5] 2024-11-27T12:25:43,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(8855): Flush requested on 817ba5bcb98df62e5ed309caecfc1068 2024-11-27T12:25:43,219 DEBUG [Thread-1508 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T12:25:43,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-27T12:25:43,220 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1ba5af43fd99,36251,1732710241631, seqNum=-1] 2024-11-27T12:25:43,220 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T12:25:43,221 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T12:25:43,221 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 817ba5bcb98df62e5ed309caecfc1068 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-27T12:25:43,224 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60150, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T12:25:43,226 DEBUG [Thread-1504 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T12:25:43,227 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5] 2024-11-27T12:25:43,228 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1ba5af43fd99,36251,1732710241631, seqNum=-1] 2024-11-27T12:25:43,228 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T12:25:43,230 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60158, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T12:25:43,232 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5] 2024-11-27T12:25:43,277 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127ecd6f9601855454bb8998460c810df21_817ba5bcb98df62e5ed309caecfc1068 is 50, key is test_row_0/A:col10/1732710343218/Put/seqid=0 2024-11-27T12:25:43,286 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:43,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60134 deadline: 1732710403276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:43,287 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:43,287 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:43,287 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:25:43,291 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:43,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60108 deadline: 1732710403277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:43,292 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:43,292 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:43,292 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:25:43,297 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:43,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60076 deadline: 1732710403280, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:43,298 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:43,297 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:43,298 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:43,298 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:25:43,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60158 deadline: 1732710403285, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:43,298 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:43,298 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:43,298 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:43,298 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:25:43,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60150 deadline: 1732710403287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:43,298 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:43,298 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:43,298 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:25:43,321 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742153_1329 (size=9714) 2024-11-27T12:25:43,322 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:25:43,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-27T12:25:43,328 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127ecd6f9601855454bb8998460c810df21_817ba5bcb98df62e5ed309caecfc1068 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127ecd6f9601855454bb8998460c810df21_817ba5bcb98df62e5ed309caecfc1068 2024-11-27T12:25:43,329 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/.tmp/A/258a02271b394e0f92b422e4ffbbefa2, store: [table=TestAcidGuarantees family=A region=817ba5bcb98df62e5ed309caecfc1068] 2024-11-27T12:25:43,330 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/.tmp/A/258a02271b394e0f92b422e4ffbbefa2 is 175, key is test_row_0/A:col10/1732710343218/Put/seqid=0 2024-11-27T12:25:43,356 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742154_1330 (size=22361) 2024-11-27T12:25:43,358 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=15, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/.tmp/A/258a02271b394e0f92b422e4ffbbefa2 2024-11-27T12:25:43,365 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-27T12:25:43,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068. 2024-11-27T12:25:43,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068. as already flushing 2024-11-27T12:25:43,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068. 2024-11-27T12:25:43,366 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 817ba5bcb98df62e5ed309caecfc1068, NAME => 'TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:25:43,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 817ba5bcb98df62e5ed309caecfc1068, NAME => 'TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:25:43,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 817ba5bcb98df62e5ed309caecfc1068, NAME => 'TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 817ba5bcb98df62e5ed309caecfc1068, NAME => 'TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:25:43,391 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/.tmp/B/c90ecc5435224a2b85feb3270cd47937 is 50, key is test_row_0/B:col10/1732710343218/Put/seqid=0 2024-11-27T12:25:43,402 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742155_1331 (size=9657) 2024-11-27T12:25:43,403 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:43,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60134 deadline: 1732710403398, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:43,404 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:43,404 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:43,404 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:25:43,404 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:43,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60108 deadline: 1732710403398, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:43,404 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:43,404 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:43,404 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:25:43,413 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:43,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60076 deadline: 1732710403408, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:43,413 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:43,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60150 deadline: 1732710403409, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:43,414 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:43,414 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:43,414 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:43,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60158 deadline: 1732710403409, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:43,414 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:25:43,414 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:43,414 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:43,414 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:25:43,416 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:43,416 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:43,416 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:25:43,519 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-27T12:25:43,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068. 2024-11-27T12:25:43,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068. as already flushing 2024-11-27T12:25:43,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068. 2024-11-27T12:25:43,519 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 817ba5bcb98df62e5ed309caecfc1068, NAME => 'TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:25:43,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 817ba5bcb98df62e5ed309caecfc1068, NAME => 'TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:25:43,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 817ba5bcb98df62e5ed309caecfc1068, NAME => 'TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 817ba5bcb98df62e5ed309caecfc1068, NAME => 'TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:25:43,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-27T12:25:43,611 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:43,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60108 deadline: 1732710403608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:43,612 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:43,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60134 deadline: 1732710403608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:43,613 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:43,613 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:43,613 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:25:43,613 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:43,613 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:43,613 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:25:43,621 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:43,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60150 deadline: 1732710403618, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:43,622 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:43,622 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:43,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60158 deadline: 1732710403619, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:43,622 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:43,622 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:25:43,622 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:43,622 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:43,622 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:25:43,631 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:43,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60076 deadline: 1732710403627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:43,631 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:43,631 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:43,632 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:25:43,672 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-27T12:25:43,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068. 2024-11-27T12:25:43,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068. as already flushing 2024-11-27T12:25:43,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068. 2024-11-27T12:25:43,672 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 817ba5bcb98df62e5ed309caecfc1068, NAME => 'TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:25:43,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 817ba5bcb98df62e5ed309caecfc1068, NAME => 'TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:25:43,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 817ba5bcb98df62e5ed309caecfc1068, NAME => 'TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 817ba5bcb98df62e5ed309caecfc1068, NAME => 'TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:25:43,802 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/.tmp/B/c90ecc5435224a2b85feb3270cd47937 2024-11-27T12:25:43,825 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-27T12:25:43,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068. 2024-11-27T12:25:43,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068. as already flushing 2024-11-27T12:25:43,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068. 2024-11-27T12:25:43,825 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 817ba5bcb98df62e5ed309caecfc1068, NAME => 'TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:25:43,826 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 817ba5bcb98df62e5ed309caecfc1068, NAME => 'TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:25:43,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 817ba5bcb98df62e5ed309caecfc1068, NAME => 'TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 817ba5bcb98df62e5ed309caecfc1068, NAME => 'TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:25:43,830 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/.tmp/C/075f9746e10c486787b659208dfac1be is 50, key is test_row_0/C:col10/1732710343218/Put/seqid=0 2024-11-27T12:25:43,843 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742156_1332 (size=9657) 2024-11-27T12:25:43,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-27T12:25:43,919 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:43,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60108 deadline: 1732710403918, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:43,920 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:43,920 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:43,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60134 deadline: 1732710403920, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:43,920 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:43,920 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:25:43,920 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:43,921 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:43,921 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:25:43,932 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:43,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60150 deadline: 1732710403928, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:43,932 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:43,932 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:43,932 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:43,932 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:25:43,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60158 deadline: 1732710403928, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:43,932 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:43,933 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:43,933 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:25:43,944 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:43,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60076 deadline: 1732710403941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:43,945 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:43,945 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:43,945 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:25:43,978 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-27T12:25:43,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068. 2024-11-27T12:25:43,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068. as already flushing 2024-11-27T12:25:43,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068. 2024-11-27T12:25:43,979 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 817ba5bcb98df62e5ed309caecfc1068, NAME => 'TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:25:43,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 817ba5bcb98df62e5ed309caecfc1068, NAME => 'TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:25:43,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 817ba5bcb98df62e5ed309caecfc1068, NAME => 'TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 817ba5bcb98df62e5ed309caecfc1068, NAME => 'TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:25:44,134 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-27T12:25:44,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068. 2024-11-27T12:25:44,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068. as already flushing 2024-11-27T12:25:44,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068. 2024-11-27T12:25:44,134 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 817ba5bcb98df62e5ed309caecfc1068, NAME => 'TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:25:44,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 817ba5bcb98df62e5ed309caecfc1068, NAME => 'TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:25:44,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 817ba5bcb98df62e5ed309caecfc1068, NAME => 'TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 817ba5bcb98df62e5ed309caecfc1068, NAME => 'TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:25:44,244 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/.tmp/C/075f9746e10c486787b659208dfac1be 2024-11-27T12:25:44,249 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/.tmp/A/258a02271b394e0f92b422e4ffbbefa2 as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/A/258a02271b394e0f92b422e4ffbbefa2 2024-11-27T12:25:44,254 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/A/258a02271b394e0f92b422e4ffbbefa2, entries=100, sequenceid=15, filesize=21.8 K 2024-11-27T12:25:44,257 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/.tmp/B/c90ecc5435224a2b85feb3270cd47937 as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/B/c90ecc5435224a2b85feb3270cd47937 2024-11-27T12:25:44,262 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/B/c90ecc5435224a2b85feb3270cd47937, entries=100, sequenceid=15, filesize=9.4 K 2024-11-27T12:25:44,264 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/.tmp/C/075f9746e10c486787b659208dfac1be as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/C/075f9746e10c486787b659208dfac1be 2024-11-27T12:25:44,270 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/C/075f9746e10c486787b659208dfac1be, entries=100, sequenceid=15, filesize=9.4 K 2024-11-27T12:25:44,271 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 817ba5bcb98df62e5ed309caecfc1068 in 1050ms, sequenceid=15, compaction requested=false 2024-11-27T12:25:44,271 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 817ba5bcb98df62e5ed309caecfc1068: 2024-11-27T12:25:44,287 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-27T12:25:44,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068. 2024-11-27T12:25:44,287 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2902): Flushing 817ba5bcb98df62e5ed309caecfc1068 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-27T12:25:44,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127a96b95dfbde54db3be06f150ec4c1412_817ba5bcb98df62e5ed309caecfc1068 is 50, key is test_row_0/A:col10/1732710343277/Put/seqid=0 2024-11-27T12:25:44,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-27T12:25:44,370 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742157_1333 (size=12154) 2024-11-27T12:25:44,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:25:44,379 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127a96b95dfbde54db3be06f150ec4c1412_817ba5bcb98df62e5ed309caecfc1068 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127a96b95dfbde54db3be06f150ec4c1412_817ba5bcb98df62e5ed309caecfc1068 2024-11-27T12:25:44,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/.tmp/A/994e597073934b64b2026363f982f497, store: [table=TestAcidGuarantees family=A region=817ba5bcb98df62e5ed309caecfc1068] 2024-11-27T12:25:44,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/.tmp/A/994e597073934b64b2026363f982f497 is 175, key is test_row_0/A:col10/1732710343277/Put/seqid=0 2024-11-27T12:25:44,415 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742158_1334 (size=30955) 2024-11-27T12:25:44,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(8855): Flush requested on 817ba5bcb98df62e5ed309caecfc1068 2024-11-27T12:25:44,431 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068. as already flushing 2024-11-27T12:25:44,445 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:44,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60150 deadline: 1732710404442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:44,446 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:44,446 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:44,446 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:44,446 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:25:44,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60134 deadline: 1732710404443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:44,446 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:44,446 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:44,446 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:25:44,451 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:44,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60108 deadline: 1732710404446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:44,452 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:44,452 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:44,452 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:25:44,456 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:44,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60158 deadline: 1732710404448, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:44,456 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:44,456 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:44,456 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:25:44,456 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:44,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60076 deadline: 1732710404448, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:44,456 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:44,456 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:44,456 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:25:44,551 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:44,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60150 deadline: 1732710404548, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:44,551 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:44,551 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:44,551 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:44,551 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:25:44,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60134 deadline: 1732710404548, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:44,552 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:44,552 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:44,552 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:25:44,559 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:44,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60108 deadline: 1732710404558, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:44,559 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:44,559 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:44,559 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:25:44,764 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:44,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60150 deadline: 1732710404758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:44,764 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:44,764 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:44,764 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:44,764 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:25:44,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60134 deadline: 1732710404758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:44,765 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:44,765 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:44,765 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:25:44,772 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:44,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60108 deadline: 1732710404768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:44,773 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:44,773 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:44,773 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:25:44,815 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=40, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/.tmp/A/994e597073934b64b2026363f982f497 2024-11-27T12:25:44,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/.tmp/B/8449fbd039ad4ba0b3b1060dbf6dc7f7 is 50, key is test_row_0/B:col10/1732710343277/Put/seqid=0 2024-11-27T12:25:44,828 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742159_1335 (size=12001) 2024-11-27T12:25:44,829 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/.tmp/B/8449fbd039ad4ba0b3b1060dbf6dc7f7 2024-11-27T12:25:44,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/.tmp/C/d7695678bc774d8382f9883e6c13afb2 is 50, key is test_row_0/C:col10/1732710343277/Put/seqid=0 2024-11-27T12:25:44,839 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742160_1336 (size=12001) 2024-11-27T12:25:45,082 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:45,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60150 deadline: 1732710405078, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:45,082 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:45,082 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:45,082 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:45,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60108 deadline: 1732710405078, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:45,082 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:25:45,083 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:45,083 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:45,083 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:45,083 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:25:45,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60134 deadline: 1732710405079, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:45,083 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:45,083 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:45,083 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:25:45,240 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/.tmp/C/d7695678bc774d8382f9883e6c13afb2 2024-11-27T12:25:45,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/.tmp/A/994e597073934b64b2026363f982f497 as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/A/994e597073934b64b2026363f982f497 2024-11-27T12:25:45,247 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/A/994e597073934b64b2026363f982f497, entries=150, sequenceid=40, filesize=30.2 K 2024-11-27T12:25:45,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/.tmp/B/8449fbd039ad4ba0b3b1060dbf6dc7f7 as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/B/8449fbd039ad4ba0b3b1060dbf6dc7f7 2024-11-27T12:25:45,252 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/B/8449fbd039ad4ba0b3b1060dbf6dc7f7, entries=150, sequenceid=40, filesize=11.7 K 2024-11-27T12:25:45,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/.tmp/C/d7695678bc774d8382f9883e6c13afb2 as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/C/d7695678bc774d8382f9883e6c13afb2 2024-11-27T12:25:45,259 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/C/d7695678bc774d8382f9883e6c13afb2, entries=150, sequenceid=40, filesize=11.7 K 2024-11-27T12:25:45,260 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 817ba5bcb98df62e5ed309caecfc1068 in 973ms, sequenceid=40, compaction requested=false 2024-11-27T12:25:45,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2603): Flush status journal for 817ba5bcb98df62e5ed309caecfc1068: 2024-11-27T12:25:45,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068. 2024-11-27T12:25:45,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=100 2024-11-27T12:25:45,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.HMaster(4169): Remote procedure done, pid=100 2024-11-27T12:25:45,263 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=100, resume processing ppid=99 2024-11-27T12:25:45,263 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=100, ppid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0490 sec 2024-11-27T12:25:45,264 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees in 2.0560 sec 2024-11-27T12:25:45,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-27T12:25:45,368 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T12:25:45,368 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-27T12:25:45,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] procedure2.ProcedureExecutor(1139): Stored pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees 2024-11-27T12:25:45,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-27T12:25:45,370 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T12:25:45,371 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T12:25:45,371 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=102, ppid=101, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T12:25:45,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(8855): Flush requested on 817ba5bcb98df62e5ed309caecfc1068 2024-11-27T12:25:45,461 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T12:25:45,461 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 817ba5bcb98df62e5ed309caecfc1068 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-27T12:25:45,469 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127a33f5304d2134290b04e23080d579125_817ba5bcb98df62e5ed309caecfc1068 is 50, key is test_row_0/A:col10/1732710344444/Put/seqid=0 2024-11-27T12:25:45,476 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742161_1337 (size=14594) 2024-11-27T12:25:45,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-27T12:25:45,523 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-27T12:25:45,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068. 2024-11-27T12:25:45,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068. as already flushing 2024-11-27T12:25:45,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068. 2024-11-27T12:25:45,523 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 817ba5bcb98df62e5ed309caecfc1068, NAME => 'TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:25:45,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 817ba5bcb98df62e5ed309caecfc1068, NAME => 'TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:25:45,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 817ba5bcb98df62e5ed309caecfc1068, NAME => 'TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 817ba5bcb98df62e5ed309caecfc1068, NAME => 'TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:25:45,579 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:45,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60158 deadline: 1732710405571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:45,579 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:45,579 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:45,579 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:25:45,582 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:45,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60076 deadline: 1732710405575, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:45,582 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:45,583 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:45,583 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:25:45,595 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:45,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60108 deadline: 1732710405588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:45,595 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:45,595 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:45,595 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:45,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60134 deadline: 1732710405588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:45,595 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:25:45,595 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:45,595 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:45,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60150 deadline: 1732710405588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:45,595 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:45,596 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:25:45,596 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:45,596 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:45,596 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:25:45,676 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-27T12:25:45,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068. 2024-11-27T12:25:45,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068. as already flushing 2024-11-27T12:25:45,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068. 2024-11-27T12:25:45,676 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 817ba5bcb98df62e5ed309caecfc1068, NAME => 'TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:25:45,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 817ba5bcb98df62e5ed309caecfc1068, NAME => 'TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:25:45,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 817ba5bcb98df62e5ed309caecfc1068, NAME => 'TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 817ba5bcb98df62e5ed309caecfc1068, NAME => 'TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:25:45,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-27T12:25:45,696 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:45,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60158 deadline: 1732710405689, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:45,697 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:45,697 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:45,697 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:45,697 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:25:45,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60076 deadline: 1732710405689, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:45,697 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:45,697 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:45,697 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:25:45,828 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-27T12:25:45,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068. 2024-11-27T12:25:45,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068. as already flushing 2024-11-27T12:25:45,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068. 2024-11-27T12:25:45,829 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 817ba5bcb98df62e5ed309caecfc1068, NAME => 'TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:25:45,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 817ba5bcb98df62e5ed309caecfc1068, NAME => 'TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:25:45,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 817ba5bcb98df62e5ed309caecfc1068, NAME => 'TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 817ba5bcb98df62e5ed309caecfc1068, NAME => 'TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:25:45,876 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:25:45,880 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127a33f5304d2134290b04e23080d579125_817ba5bcb98df62e5ed309caecfc1068 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127a33f5304d2134290b04e23080d579125_817ba5bcb98df62e5ed309caecfc1068 2024-11-27T12:25:45,881 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/.tmp/A/607c0ee7a4174c4694c7f374f5e820fc, store: [table=TestAcidGuarantees family=A region=817ba5bcb98df62e5ed309caecfc1068] 2024-11-27T12:25:45,882 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/.tmp/A/607c0ee7a4174c4694c7f374f5e820fc is 175, key is test_row_0/A:col10/1732710344444/Put/seqid=0 2024-11-27T12:25:45,885 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742162_1338 (size=39549) 2024-11-27T12:25:45,888 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=52, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/.tmp/A/607c0ee7a4174c4694c7f374f5e820fc 2024-11-27T12:25:45,915 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:45,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60158 deadline: 1732710405908, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:45,915 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:45,915 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:45,916 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:45,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60076 deadline: 1732710405908, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:45,916 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:25:45,916 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:45,916 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:45,916 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:25:45,918 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/.tmp/B/0190efb6011340f0b3305ffe36521cf3 is 50, key is test_row_0/B:col10/1732710344444/Put/seqid=0 2024-11-27T12:25:45,944 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742163_1339 (size=12001) 2024-11-27T12:25:45,945 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/.tmp/B/0190efb6011340f0b3305ffe36521cf3 2024-11-27T12:25:45,960 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/.tmp/C/796725ff60204c8b9c1337c461d290ef is 50, key is test_row_0/C:col10/1732710344444/Put/seqid=0 2024-11-27T12:25:45,970 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742164_1340 (size=12001) 2024-11-27T12:25:45,971 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/.tmp/C/796725ff60204c8b9c1337c461d290ef 2024-11-27T12:25:45,981 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-27T12:25:45,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068. 2024-11-27T12:25:45,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068. as already flushing 2024-11-27T12:25:45,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068. 2024-11-27T12:25:45,982 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 817ba5bcb98df62e5ed309caecfc1068, NAME => 'TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:25:45,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 817ba5bcb98df62e5ed309caecfc1068, NAME => 'TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:25:45,985 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/.tmp/A/607c0ee7a4174c4694c7f374f5e820fc as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/A/607c0ee7a4174c4694c7f374f5e820fc 2024-11-27T12:25:45,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 817ba5bcb98df62e5ed309caecfc1068, NAME => 'TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 817ba5bcb98df62e5ed309caecfc1068, NAME => 'TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:25:45,991 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/A/607c0ee7a4174c4694c7f374f5e820fc, entries=200, sequenceid=52, filesize=38.6 K 2024-11-27T12:25:45,992 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/.tmp/B/0190efb6011340f0b3305ffe36521cf3 as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/B/0190efb6011340f0b3305ffe36521cf3 2024-11-27T12:25:45,996 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/B/0190efb6011340f0b3305ffe36521cf3, entries=150, sequenceid=52, filesize=11.7 K 2024-11-27T12:25:45,997 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/.tmp/C/796725ff60204c8b9c1337c461d290ef as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/C/796725ff60204c8b9c1337c461d290ef 2024-11-27T12:25:45,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-27T12:25:46,003 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/C/796725ff60204c8b9c1337c461d290ef, entries=150, sequenceid=52, filesize=11.7 K 2024-11-27T12:25:46,004 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 817ba5bcb98df62e5ed309caecfc1068 in 542ms, sequenceid=52, compaction requested=true 2024-11-27T12:25:46,004 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 817ba5bcb98df62e5ed309caecfc1068: 2024-11-27T12:25:46,004 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 817ba5bcb98df62e5ed309caecfc1068:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T12:25:46,004 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T12:25:46,004 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T12:25:46,004 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 817ba5bcb98df62e5ed309caecfc1068:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T12:25:46,004 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T12:25:46,004 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T12:25:46,004 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 817ba5bcb98df62e5ed309caecfc1068:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T12:25:46,004 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T12:25:46,005 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 92865 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T12:25:46,005 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HStore(1541): 817ba5bcb98df62e5ed309caecfc1068/A is initiating minor compaction (all files) 2024-11-27T12:25:46,005 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T12:25:46,005 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 817ba5bcb98df62e5ed309caecfc1068/A in TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068. 2024-11-27T12:25:46,005 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HStore(1541): 817ba5bcb98df62e5ed309caecfc1068/B is initiating minor compaction (all files) 2024-11-27T12:25:46,005 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 817ba5bcb98df62e5ed309caecfc1068/B in TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068. 2024-11-27T12:25:46,005 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/A/258a02271b394e0f92b422e4ffbbefa2, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/A/994e597073934b64b2026363f982f497, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/A/607c0ee7a4174c4694c7f374f5e820fc] into tmpdir=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/.tmp, totalSize=90.7 K 2024-11-27T12:25:46,005 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068. 2024-11-27T12:25:46,005 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/B/c90ecc5435224a2b85feb3270cd47937, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/B/8449fbd039ad4ba0b3b1060dbf6dc7f7, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/B/0190efb6011340f0b3305ffe36521cf3] into tmpdir=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/.tmp, totalSize=32.9 K 2024-11-27T12:25:46,005 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068. files: [hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/A/258a02271b394e0f92b422e4ffbbefa2, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/A/994e597073934b64b2026363f982f497, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/A/607c0ee7a4174c4694c7f374f5e820fc] 2024-11-27T12:25:46,006 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.Compactor(225): Compacting c90ecc5435224a2b85feb3270cd47937, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1732710343218 2024-11-27T12:25:46,006 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] compactions.Compactor(225): Compacting 258a02271b394e0f92b422e4ffbbefa2, keycount=100, bloomtype=ROW, size=21.8 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1732710343218 2024-11-27T12:25:46,006 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.Compactor(225): Compacting 8449fbd039ad4ba0b3b1060dbf6dc7f7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1732710343266 2024-11-27T12:25:46,006 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] compactions.Compactor(225): Compacting 994e597073934b64b2026363f982f497, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1732710343266 2024-11-27T12:25:46,006 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.Compactor(225): Compacting 0190efb6011340f0b3305ffe36521cf3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1732710344430 2024-11-27T12:25:46,006 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] compactions.Compactor(225): Compacting 607c0ee7a4174c4694c7f374f5e820fc, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1732710344430 2024-11-27T12:25:46,012 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=817ba5bcb98df62e5ed309caecfc1068] 2024-11-27T12:25:46,013 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241127023cd1247b4f4d9897f456f8c8a9a0d9_817ba5bcb98df62e5ed309caecfc1068 store=[table=TestAcidGuarantees family=A region=817ba5bcb98df62e5ed309caecfc1068] 2024-11-27T12:25:46,014 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 817ba5bcb98df62e5ed309caecfc1068#B#compaction#289 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-27T12:25:46,014 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/.tmp/B/8135cb7ee15943d8b70edc4caa48092b is 50, key is test_row_0/B:col10/1732710344444/Put/seqid=0 2024-11-27T12:25:46,015 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241127023cd1247b4f4d9897f456f8c8a9a0d9_817ba5bcb98df62e5ed309caecfc1068, store=[table=TestAcidGuarantees family=A region=817ba5bcb98df62e5ed309caecfc1068] 2024-11-27T12:25:46,015 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127023cd1247b4f4d9897f456f8c8a9a0d9_817ba5bcb98df62e5ed309caecfc1068 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=817ba5bcb98df62e5ed309caecfc1068] 2024-11-27T12:25:46,022 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742166_1342 (size=4469) 2024-11-27T12:25:46,025 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742165_1341 (size=12104) 2024-11-27T12:25:46,032 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/.tmp/B/8135cb7ee15943d8b70edc4caa48092b as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/B/8135cb7ee15943d8b70edc4caa48092b 2024-11-27T12:25:46,039 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 817ba5bcb98df62e5ed309caecfc1068/B of 817ba5bcb98df62e5ed309caecfc1068 into 8135cb7ee15943d8b70edc4caa48092b(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T12:25:46,039 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 817ba5bcb98df62e5ed309caecfc1068: 2024-11-27T12:25:46,039 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., storeName=817ba5bcb98df62e5ed309caecfc1068/B, priority=13, startTime=1732710346004; duration=0sec 2024-11-27T12:25:46,039 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T12:25:46,039 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 817ba5bcb98df62e5ed309caecfc1068:B 2024-11-27T12:25:46,039 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T12:25:46,040 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T12:25:46,040 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HStore(1541): 817ba5bcb98df62e5ed309caecfc1068/C is initiating minor compaction (all files) 2024-11-27T12:25:46,040 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 817ba5bcb98df62e5ed309caecfc1068/C in TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068. 2024-11-27T12:25:46,041 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/C/075f9746e10c486787b659208dfac1be, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/C/d7695678bc774d8382f9883e6c13afb2, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/C/796725ff60204c8b9c1337c461d290ef] into tmpdir=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/.tmp, totalSize=32.9 K 2024-11-27T12:25:46,041 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.Compactor(225): Compacting 075f9746e10c486787b659208dfac1be, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1732710343218 2024-11-27T12:25:46,042 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.Compactor(225): Compacting d7695678bc774d8382f9883e6c13afb2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1732710343266 2024-11-27T12:25:46,042 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.Compactor(225): Compacting 796725ff60204c8b9c1337c461d290ef, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1732710344430 2024-11-27T12:25:46,051 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 817ba5bcb98df62e5ed309caecfc1068#C#compaction#290 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-27T12:25:46,051 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/.tmp/C/c8fcd07d85034991a43cfca6db974b92 is 50, key is test_row_0/C:col10/1732710344444/Put/seqid=0 2024-11-27T12:25:46,055 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742167_1343 (size=12104) 2024-11-27T12:25:46,137 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-27T12:25:46,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068. 2024-11-27T12:25:46,138 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2902): Flushing 817ba5bcb98df62e5ed309caecfc1068 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-27T12:25:46,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411276da5353fb6e04c649432ae38e0a14e45_817ba5bcb98df62e5ed309caecfc1068 is 50, key is test_row_0/A:col10/1732710345573/Put/seqid=0 2024-11-27T12:25:46,148 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742168_1344 (size=12154) 2024-11-27T12:25:46,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(8855): Flush requested on 817ba5bcb98df62e5ed309caecfc1068 2024-11-27T12:25:46,223 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068. as already flushing 2024-11-27T12:25:46,254 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-27T12:25:46,269 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:46,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60158 deadline: 1732710406259, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:46,269 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:46,269 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:46,270 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:25:46,271 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:46,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60076 deadline: 1732710406265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:46,272 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:46,272 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:46,272 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:25:46,384 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:46,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60158 deadline: 1732710406378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:46,384 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:46,384 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:46,384 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:46,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60076 deadline: 1732710406378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:46,384 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:25:46,384 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:46,384 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:46,384 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:25:46,423 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 817ba5bcb98df62e5ed309caecfc1068#A#compaction#288 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T12:25:46,423 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/.tmp/A/4827b378778f4effad509797bdf5d641 is 175, key is test_row_0/A:col10/1732710344444/Put/seqid=0 2024-11-27T12:25:46,426 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742169_1345 (size=31058) 2024-11-27T12:25:46,459 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/.tmp/C/c8fcd07d85034991a43cfca6db974b92 as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/C/c8fcd07d85034991a43cfca6db974b92 2024-11-27T12:25:46,464 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 817ba5bcb98df62e5ed309caecfc1068/C of 817ba5bcb98df62e5ed309caecfc1068 into c8fcd07d85034991a43cfca6db974b92(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T12:25:46,464 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 817ba5bcb98df62e5ed309caecfc1068: 2024-11-27T12:25:46,464 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., storeName=817ba5bcb98df62e5ed309caecfc1068/C, priority=13, startTime=1732710346004; duration=0sec 2024-11-27T12:25:46,464 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T12:25:46,464 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 817ba5bcb98df62e5ed309caecfc1068:C 2024-11-27T12:25:46,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-27T12:25:46,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:25:46,553 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411276da5353fb6e04c649432ae38e0a14e45_817ba5bcb98df62e5ed309caecfc1068 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411276da5353fb6e04c649432ae38e0a14e45_817ba5bcb98df62e5ed309caecfc1068 2024-11-27T12:25:46,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/.tmp/A/ea9002f10f3c486eb77eb08b1d87af15, store: [table=TestAcidGuarantees family=A region=817ba5bcb98df62e5ed309caecfc1068] 2024-11-27T12:25:46,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/.tmp/A/ea9002f10f3c486eb77eb08b1d87af15 is 175, key is test_row_0/A:col10/1732710345573/Put/seqid=0 2024-11-27T12:25:46,557 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742170_1346 (size=30955) 2024-11-27T12:25:46,593 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:46,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60158 deadline: 1732710406588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:46,593 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:46,593 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:46,593 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:46,593 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:25:46,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60076 deadline: 1732710406588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:46,594 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:46,594 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:46,594 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:25:46,604 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:46,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60150 deadline: 1732710406598, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:46,604 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 1ba5af43fd99,36251,1732710241631 for 'test_row_0' in 817ba5bcb98df62e5ed309caecfc1068 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2162 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T12:25:46,605 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:46,605 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:46,605 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:25:46,613 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:46,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60108 deadline: 1732710406608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:46,613 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:46,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60134 deadline: 1732710406608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:46,613 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 1ba5af43fd99,36251,1732710241631 for 'test_row_0' in 817ba5bcb98df62e5ed309caecfc1068 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2167 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T12:25:46,614 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:46,614 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:46,614 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:25:46,614 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 1ba5af43fd99,36251,1732710241631 for 'test_row_0' in 817ba5bcb98df62e5ed309caecfc1068 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2171 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T12:25:46,614 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:46,614 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:46,614 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:25:46,831 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/.tmp/A/4827b378778f4effad509797bdf5d641 as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/A/4827b378778f4effad509797bdf5d641 2024-11-27T12:25:46,834 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 817ba5bcb98df62e5ed309caecfc1068/A of 817ba5bcb98df62e5ed309caecfc1068 into 4827b378778f4effad509797bdf5d641(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T12:25:46,835 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 817ba5bcb98df62e5ed309caecfc1068: 2024-11-27T12:25:46,835 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., storeName=817ba5bcb98df62e5ed309caecfc1068/A, priority=13, startTime=1732710346004; duration=0sec 2024-11-27T12:25:46,835 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T12:25:46,835 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 817ba5bcb98df62e5ed309caecfc1068:A 2024-11-27T12:25:46,902 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:46,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60158 deadline: 1732710406897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:46,903 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:46,903 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:46,903 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:46,903 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:25:46,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60076 deadline: 1732710406898, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:46,903 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:46,903 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:46,903 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:25:46,958 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=77, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/.tmp/A/ea9002f10f3c486eb77eb08b1d87af15 2024-11-27T12:25:46,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/.tmp/B/53450ea59c9a43d2af06c6c97d0f3321 is 50, key is test_row_0/B:col10/1732710345573/Put/seqid=0 2024-11-27T12:25:46,969 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742171_1347 (size=12001) 2024-11-27T12:25:47,370 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=77 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/.tmp/B/53450ea59c9a43d2af06c6c97d0f3321 2024-11-27T12:25:47,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/.tmp/C/5cdd1c6857d2443ea2992b9a022d5c65 is 50, key is test_row_0/C:col10/1732710345573/Put/seqid=0 2024-11-27T12:25:47,379 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742172_1348 (size=12001) 2024-11-27T12:25:47,423 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:47,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60158 deadline: 1732710407418, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:47,424 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:47,424 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:47,424 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:25:47,424 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:47,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60076 deadline: 1732710407418, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:47,424 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:47,424 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:47,424 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:25:47,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-27T12:25:47,780 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=77 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/.tmp/C/5cdd1c6857d2443ea2992b9a022d5c65 2024-11-27T12:25:47,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/.tmp/A/ea9002f10f3c486eb77eb08b1d87af15 as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/A/ea9002f10f3c486eb77eb08b1d87af15 2024-11-27T12:25:47,797 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/A/ea9002f10f3c486eb77eb08b1d87af15, entries=150, sequenceid=77, filesize=30.2 K 2024-11-27T12:25:47,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/.tmp/B/53450ea59c9a43d2af06c6c97d0f3321 as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/B/53450ea59c9a43d2af06c6c97d0f3321 2024-11-27T12:25:47,807 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/B/53450ea59c9a43d2af06c6c97d0f3321, entries=150, sequenceid=77, filesize=11.7 K 2024-11-27T12:25:47,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/.tmp/C/5cdd1c6857d2443ea2992b9a022d5c65 as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/C/5cdd1c6857d2443ea2992b9a022d5c65 2024-11-27T12:25:47,817 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/C/5cdd1c6857d2443ea2992b9a022d5c65, entries=150, sequenceid=77, filesize=11.7 K 2024-11-27T12:25:47,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.StoreScanner(1005): StoreScanner already closing. There is no need to updateReaders 2024-11-27T12:25:47,818 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 817ba5bcb98df62e5ed309caecfc1068 in 1680ms, sequenceid=77, compaction requested=false 2024-11-27T12:25:47,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2603): Flush status journal for 817ba5bcb98df62e5ed309caecfc1068: 2024-11-27T12:25:47,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068. 2024-11-27T12:25:47,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=102 2024-11-27T12:25:47,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.HMaster(4169): Remote procedure done, pid=102 2024-11-27T12:25:47,822 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=102, resume processing ppid=101 2024-11-27T12:25:47,822 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=102, ppid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.4490 sec 2024-11-27T12:25:47,823 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees in 2.4540 sec 2024-11-27T12:25:48,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(8855): Flush requested on 817ba5bcb98df62e5ed309caecfc1068 2024-11-27T12:25:48,435 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T12:25:48,436 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 817ba5bcb98df62e5ed309caecfc1068 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-27T12:25:48,468 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127f250a53b2beb41849231825735062507_817ba5bcb98df62e5ed309caecfc1068 is 50, key is test_row_0/A:col10/1732710346263/Put/seqid=0 2024-11-27T12:25:48,502 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742173_1349 (size=14594) 2024-11-27T12:25:48,503 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:25:48,514 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127f250a53b2beb41849231825735062507_817ba5bcb98df62e5ed309caecfc1068 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127f250a53b2beb41849231825735062507_817ba5bcb98df62e5ed309caecfc1068 2024-11-27T12:25:48,515 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/.tmp/A/baa90092ebbe4b69b418c678b4d78644, store: [table=TestAcidGuarantees family=A region=817ba5bcb98df62e5ed309caecfc1068] 2024-11-27T12:25:48,515 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/.tmp/A/baa90092ebbe4b69b418c678b4d78644 is 175, key is test_row_0/A:col10/1732710346263/Put/seqid=0 2024-11-27T12:25:48,526 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742174_1350 (size=39549) 2024-11-27T12:25:48,527 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=92, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/.tmp/A/baa90092ebbe4b69b418c678b4d78644 2024-11-27T12:25:48,537 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/.tmp/B/80af0fb64dfb4d81b80a23ff6e58f885 is 50, key is test_row_0/B:col10/1732710346263/Put/seqid=0 2024-11-27T12:25:48,543 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742175_1351 (size=12001) 2024-11-27T12:25:48,558 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:48,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60076 deadline: 1732710408554, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:48,558 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:48,558 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:48,558 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:25:48,567 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:48,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60158 deadline: 1732710408555, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:48,567 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:48,567 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:48,567 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:25:48,625 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:48,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60150 deadline: 1732710408618, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:48,626 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 1ba5af43fd99,36251,1732710241631 for 'test_row_0' in 817ba5bcb98df62e5ed309caecfc1068 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4183 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T12:25:48,626 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:48,626 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:48,626 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:25:48,642 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:48,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60108 deadline: 1732710408638, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:48,643 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:48,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60134 deadline: 1732710408638, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:48,643 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 1ba5af43fd99,36251,1732710241631 for 'test_row_0' in 817ba5bcb98df62e5ed309caecfc1068 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4197 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T12:25:48,643 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:48,643 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:48,643 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:25:48,643 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 1ba5af43fd99,36251,1732710241631 for 'test_row_0' in 817ba5bcb98df62e5ed309caecfc1068 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4200 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T12:25:48,644 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:48,644 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:48,644 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:25:48,678 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:48,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60076 deadline: 1732710408668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:48,678 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:48,678 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:48,678 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:25:48,683 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:48,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60158 deadline: 1732710408678, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:48,683 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:48,683 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:48,683 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:25:48,897 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:48,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60076 deadline: 1732710408888, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:48,900 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:48,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60158 deadline: 1732710408888, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:48,900 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:48,900 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:48,900 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:48,900 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:48,900 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:25:48,900 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:25:48,944 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=92 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/.tmp/B/80af0fb64dfb4d81b80a23ff6e58f885 2024-11-27T12:25:48,974 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/.tmp/C/fcccf1ce41c947f3a4f6c6186ee2537f is 50, key is test_row_0/C:col10/1732710346263/Put/seqid=0 2024-11-27T12:25:48,987 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742176_1352 (size=12001) 2024-11-27T12:25:49,213 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:49,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60076 deadline: 1732710409208, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:49,213 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:49,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60158 deadline: 1732710409208, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:49,213 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:49,213 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:49,213 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:25:49,213 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:49,213 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:49,213 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:25:49,388 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=92 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/.tmp/C/fcccf1ce41c947f3a4f6c6186ee2537f 2024-11-27T12:25:49,393 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/.tmp/A/baa90092ebbe4b69b418c678b4d78644 as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/A/baa90092ebbe4b69b418c678b4d78644 2024-11-27T12:25:49,402 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/A/baa90092ebbe4b69b418c678b4d78644, entries=200, sequenceid=92, filesize=38.6 K 2024-11-27T12:25:49,403 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/.tmp/B/80af0fb64dfb4d81b80a23ff6e58f885 as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/B/80af0fb64dfb4d81b80a23ff6e58f885 2024-11-27T12:25:49,418 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/B/80af0fb64dfb4d81b80a23ff6e58f885, entries=150, sequenceid=92, filesize=11.7 K 2024-11-27T12:25:49,419 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/.tmp/C/fcccf1ce41c947f3a4f6c6186ee2537f as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/C/fcccf1ce41c947f3a4f6c6186ee2537f 2024-11-27T12:25:49,423 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/C/fcccf1ce41c947f3a4f6c6186ee2537f, entries=150, sequenceid=92, filesize=11.7 K 2024-11-27T12:25:49,427 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 817ba5bcb98df62e5ed309caecfc1068 in 992ms, sequenceid=92, compaction requested=true 2024-11-27T12:25:49,427 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 817ba5bcb98df62e5ed309caecfc1068: 2024-11-27T12:25:49,427 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 817ba5bcb98df62e5ed309caecfc1068:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T12:25:49,427 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T12:25:49,427 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T12:25:49,427 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 817ba5bcb98df62e5ed309caecfc1068:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T12:25:49,427 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T12:25:49,427 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 817ba5bcb98df62e5ed309caecfc1068:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T12:25:49,427 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-27T12:25:49,427 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T12:25:49,429 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T12:25:49,429 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HStore(1541): 817ba5bcb98df62e5ed309caecfc1068/B is initiating minor compaction (all files) 2024-11-27T12:25:49,429 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 817ba5bcb98df62e5ed309caecfc1068/B in TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068. 2024-11-27T12:25:49,429 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/B/8135cb7ee15943d8b70edc4caa48092b, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/B/53450ea59c9a43d2af06c6c97d0f3321, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/B/80af0fb64dfb4d81b80a23ff6e58f885] into tmpdir=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/.tmp, totalSize=35.3 K 2024-11-27T12:25:49,429 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101562 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T12:25:49,429 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HStore(1541): 817ba5bcb98df62e5ed309caecfc1068/A is initiating minor compaction (all files) 2024-11-27T12:25:49,429 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 817ba5bcb98df62e5ed309caecfc1068/A in TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068. 2024-11-27T12:25:49,429 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/A/4827b378778f4effad509797bdf5d641, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/A/ea9002f10f3c486eb77eb08b1d87af15, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/A/baa90092ebbe4b69b418c678b4d78644] into tmpdir=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/.tmp, totalSize=99.2 K 2024-11-27T12:25:49,430 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068. 2024-11-27T12:25:49,430 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068. files: [hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/A/4827b378778f4effad509797bdf5d641, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/A/ea9002f10f3c486eb77eb08b1d87af15, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/A/baa90092ebbe4b69b418c678b4d78644] 2024-11-27T12:25:49,430 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.Compactor(225): Compacting 8135cb7ee15943d8b70edc4caa48092b, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1732710344430 2024-11-27T12:25:49,430 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4827b378778f4effad509797bdf5d641, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1732710344430 2024-11-27T12:25:49,431 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.Compactor(225): Compacting 53450ea59c9a43d2af06c6c97d0f3321, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1732710345546 2024-11-27T12:25:49,431 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] compactions.Compactor(225): Compacting ea9002f10f3c486eb77eb08b1d87af15, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1732710345546 2024-11-27T12:25:49,431 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.Compactor(225): Compacting 80af0fb64dfb4d81b80a23ff6e58f885, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1732710346255 2024-11-27T12:25:49,432 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] compactions.Compactor(225): Compacting baa90092ebbe4b69b418c678b4d78644, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1732710346243 2024-11-27T12:25:49,443 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 817ba5bcb98df62e5ed309caecfc1068#B#compaction#297 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T12:25:49,443 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/.tmp/B/52ca6d4fd2624927a2898f9b8f1a1e31 is 50, key is test_row_0/B:col10/1732710346263/Put/seqid=0 2024-11-27T12:25:49,461 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=817ba5bcb98df62e5ed309caecfc1068] 2024-11-27T12:25:49,484 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411276268cd756602465082adf49c17af4638_817ba5bcb98df62e5ed309caecfc1068 store=[table=TestAcidGuarantees family=A region=817ba5bcb98df62e5ed309caecfc1068] 2024-11-27T12:25:49,486 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411276268cd756602465082adf49c17af4638_817ba5bcb98df62e5ed309caecfc1068, store=[table=TestAcidGuarantees family=A region=817ba5bcb98df62e5ed309caecfc1068] 2024-11-27T12:25:49,486 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411276268cd756602465082adf49c17af4638_817ba5bcb98df62e5ed309caecfc1068 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=817ba5bcb98df62e5ed309caecfc1068] 2024-11-27T12:25:49,519 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742177_1353 (size=12207) 2024-11-27T12:25:49,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-27T12:25:49,539 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T12:25:49,539 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-27T12:25:49,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] procedure2.ProcedureExecutor(1139): Stored pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees 2024-11-27T12:25:49,542 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T12:25:49,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-27T12:25:49,543 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T12:25:49,543 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=104, ppid=103, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T12:25:49,544 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742178_1354 (size=4469) 2024-11-27T12:25:49,546 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 817ba5bcb98df62e5ed309caecfc1068#A#compaction#298 average throughput is 0.29 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T12:25:49,546 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/.tmp/A/40e5f80a08ee4b9993dff964912a7a7e is 175, key is test_row_0/A:col10/1732710346263/Put/seqid=0 2024-11-27T12:25:49,581 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742179_1355 (size=31161) 2024-11-27T12:25:49,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-27T12:25:49,694 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-27T12:25:49,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068. 2024-11-27T12:25:49,695 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2902): Flushing 817ba5bcb98df62e5ed309caecfc1068 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-27T12:25:49,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127166ec1798f354071b9b8974964ec0efc_817ba5bcb98df62e5ed309caecfc1068 is 50, key is test_row_0/A:col10/1732710348553/Put/seqid=0 2024-11-27T12:25:49,725 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742180_1356 (size=12154) 2024-11-27T12:25:49,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:25:49,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(8855): Flush requested on 817ba5bcb98df62e5ed309caecfc1068 2024-11-27T12:25:49,727 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068. as already flushing 2024-11-27T12:25:49,733 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127166ec1798f354071b9b8974964ec0efc_817ba5bcb98df62e5ed309caecfc1068 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127166ec1798f354071b9b8974964ec0efc_817ba5bcb98df62e5ed309caecfc1068 2024-11-27T12:25:49,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/.tmp/A/8889c0115b8a49eabfe31a6e3fd0c03a, store: [table=TestAcidGuarantees family=A region=817ba5bcb98df62e5ed309caecfc1068] 2024-11-27T12:25:49,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/.tmp/A/8889c0115b8a49eabfe31a6e3fd0c03a is 175, key is test_row_0/A:col10/1732710348553/Put/seqid=0 2024-11-27T12:25:49,749 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742181_1357 (size=30955) 2024-11-27T12:25:49,749 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=115, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/.tmp/A/8889c0115b8a49eabfe31a6e3fd0c03a 2024-11-27T12:25:49,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/.tmp/B/d01946b594b642509cc52b66195c09ea is 50, key is test_row_0/B:col10/1732710348553/Put/seqid=0 2024-11-27T12:25:49,776 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742182_1358 (size=12001) 2024-11-27T12:25:49,776 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:49,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60158 deadline: 1732710409764, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:49,778 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:49,778 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:49,778 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:25:49,778 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=115 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/.tmp/B/d01946b594b642509cc52b66195c09ea 2024-11-27T12:25:49,780 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:49,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60076 deadline: 1732710409769, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:49,781 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:49,781 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:49,781 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:25:49,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/.tmp/C/92bb4ff9bdaf4e2a949f294c39774a8b is 50, key is test_row_0/C:col10/1732710348553/Put/seqid=0 2024-11-27T12:25:49,822 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742183_1359 (size=12001) 2024-11-27T12:25:49,823 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=115 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/.tmp/C/92bb4ff9bdaf4e2a949f294c39774a8b 2024-11-27T12:25:49,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/.tmp/A/8889c0115b8a49eabfe31a6e3fd0c03a as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/A/8889c0115b8a49eabfe31a6e3fd0c03a 2024-11-27T12:25:49,836 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/A/8889c0115b8a49eabfe31a6e3fd0c03a, entries=150, sequenceid=115, filesize=30.2 K 2024-11-27T12:25:49,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/.tmp/B/d01946b594b642509cc52b66195c09ea as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/B/d01946b594b642509cc52b66195c09ea 2024-11-27T12:25:49,845 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/B/d01946b594b642509cc52b66195c09ea, entries=150, sequenceid=115, filesize=11.7 K 2024-11-27T12:25:49,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/.tmp/C/92bb4ff9bdaf4e2a949f294c39774a8b as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/C/92bb4ff9bdaf4e2a949f294c39774a8b 2024-11-27T12:25:49,852 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/C/92bb4ff9bdaf4e2a949f294c39774a8b, entries=150, sequenceid=115, filesize=11.7 K 2024-11-27T12:25:49,853 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for 817ba5bcb98df62e5ed309caecfc1068 in 158ms, sequenceid=115, compaction requested=true 2024-11-27T12:25:49,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2603): Flush status journal for 817ba5bcb98df62e5ed309caecfc1068: 2024-11-27T12:25:49,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068. 2024-11-27T12:25:49,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=104 2024-11-27T12:25:49,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.HMaster(4169): Remote procedure done, pid=104 2024-11-27T12:25:49,856 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=104, resume processing ppid=103 2024-11-27T12:25:49,856 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=104, ppid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 312 msec 2024-11-27T12:25:49,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-27T12:25:49,858 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees in 317 msec 2024-11-27T12:25:49,889 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T12:25:49,890 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 817ba5bcb98df62e5ed309caecfc1068 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-27T12:25:49,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(8855): Flush requested on 817ba5bcb98df62e5ed309caecfc1068 2024-11-27T12:25:49,899 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127d45b03a57566468cac57ab494ce0d378_817ba5bcb98df62e5ed309caecfc1068 is 50, key is test_row_0/A:col10/1732710349728/Put/seqid=0 2024-11-27T12:25:49,909 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742184_1360 (size=14644) 2024-11-27T12:25:49,910 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:25:49,914 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127d45b03a57566468cac57ab494ce0d378_817ba5bcb98df62e5ed309caecfc1068 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127d45b03a57566468cac57ab494ce0d378_817ba5bcb98df62e5ed309caecfc1068 2024-11-27T12:25:49,915 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/.tmp/A/969e6cbf7ac643aebcd72d6b8127478e, store: [table=TestAcidGuarantees family=A region=817ba5bcb98df62e5ed309caecfc1068] 2024-11-27T12:25:49,916 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/.tmp/A/969e6cbf7ac643aebcd72d6b8127478e is 175, key is test_row_0/A:col10/1732710349728/Put/seqid=0 2024-11-27T12:25:49,919 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742185_1361 (size=39599) 2024-11-27T12:25:49,923 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=129, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/.tmp/A/969e6cbf7ac643aebcd72d6b8127478e 2024-11-27T12:25:49,932 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/.tmp/B/52ca6d4fd2624927a2898f9b8f1a1e31 as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/B/52ca6d4fd2624927a2898f9b8f1a1e31 2024-11-27T12:25:49,954 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 817ba5bcb98df62e5ed309caecfc1068/B of 817ba5bcb98df62e5ed309caecfc1068 into 52ca6d4fd2624927a2898f9b8f1a1e31(size=11.9 K), total size for store is 23.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T12:25:49,954 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 817ba5bcb98df62e5ed309caecfc1068: 2024-11-27T12:25:49,954 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., storeName=817ba5bcb98df62e5ed309caecfc1068/B, priority=13, startTime=1732710349427; duration=0sec 2024-11-27T12:25:49,954 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T12:25:49,954 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 817ba5bcb98df62e5ed309caecfc1068:B 2024-11-27T12:25:49,954 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-27T12:25:49,956 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-27T12:25:49,956 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HStore(1541): 817ba5bcb98df62e5ed309caecfc1068/C is initiating minor compaction (all files) 2024-11-27T12:25:49,956 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 817ba5bcb98df62e5ed309caecfc1068/C in TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068. 2024-11-27T12:25:49,957 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/C/c8fcd07d85034991a43cfca6db974b92, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/C/5cdd1c6857d2443ea2992b9a022d5c65, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/C/fcccf1ce41c947f3a4f6c6186ee2537f, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/C/92bb4ff9bdaf4e2a949f294c39774a8b] into tmpdir=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/.tmp, totalSize=47.0 K 2024-11-27T12:25:49,957 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.Compactor(225): Compacting c8fcd07d85034991a43cfca6db974b92, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1732710344430 2024-11-27T12:25:49,957 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.Compactor(225): Compacting 5cdd1c6857d2443ea2992b9a022d5c65, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1732710345546 2024-11-27T12:25:49,957 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.Compactor(225): Compacting fcccf1ce41c947f3a4f6c6186ee2537f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1732710346255 2024-11-27T12:25:49,963 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.Compactor(225): Compacting 92bb4ff9bdaf4e2a949f294c39774a8b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1732710348532 2024-11-27T12:25:49,974 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/.tmp/B/7bbe02f709b74eb08cf8016be0748378 is 50, key is test_row_0/B:col10/1732710349728/Put/seqid=0 2024-11-27T12:25:49,982 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 817ba5bcb98df62e5ed309caecfc1068#C#compaction#304 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T12:25:49,983 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/.tmp/C/06b2c9f79fd940c9965d97cf9b259ab4 is 50, key is test_row_0/C:col10/1732710348553/Put/seqid=0 2024-11-27T12:25:49,987 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/.tmp/A/40e5f80a08ee4b9993dff964912a7a7e as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/A/40e5f80a08ee4b9993dff964912a7a7e 2024-11-27T12:25:49,993 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 817ba5bcb98df62e5ed309caecfc1068/A of 817ba5bcb98df62e5ed309caecfc1068 into 40e5f80a08ee4b9993dff964912a7a7e(size=30.4 K), total size for store is 60.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T12:25:49,993 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 817ba5bcb98df62e5ed309caecfc1068: 2024-11-27T12:25:49,993 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., storeName=817ba5bcb98df62e5ed309caecfc1068/A, priority=13, startTime=1732710349427; duration=0sec 2024-11-27T12:25:49,993 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T12:25:49,993 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 817ba5bcb98df62e5ed309caecfc1068:A 2024-11-27T12:25:50,003 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:50,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60076 deadline: 1732710409992, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:50,004 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:50,004 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:50,004 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:25:50,006 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742186_1362 (size=12051) 2024-11-27T12:25:50,009 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=129 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/.tmp/B/7bbe02f709b74eb08cf8016be0748378 2024-11-27T12:25:50,010 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742187_1363 (size=12241) 2024-11-27T12:25:50,022 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:50,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60158 deadline: 1732710410003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:50,023 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:50,023 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:50,023 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:25:50,030 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/.tmp/C/06b2c9f79fd940c9965d97cf9b259ab4 as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/C/06b2c9f79fd940c9965d97cf9b259ab4 2024-11-27T12:25:50,036 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 817ba5bcb98df62e5ed309caecfc1068/C of 817ba5bcb98df62e5ed309caecfc1068 into 06b2c9f79fd940c9965d97cf9b259ab4(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T12:25:50,036 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 817ba5bcb98df62e5ed309caecfc1068: 2024-11-27T12:25:50,036 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., storeName=817ba5bcb98df62e5ed309caecfc1068/C, priority=12, startTime=1732710349427; duration=0sec 2024-11-27T12:25:50,036 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T12:25:50,036 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 817ba5bcb98df62e5ed309caecfc1068:C 2024-11-27T12:25:50,047 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/.tmp/C/8e22b3175f5e4838bea3f7f13de4e688 is 50, key is test_row_0/C:col10/1732710349728/Put/seqid=0 2024-11-27T12:25:50,058 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742188_1364 (size=12051) 2024-11-27T12:25:50,116 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:50,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60076 deadline: 1732710410108, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:50,118 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:50,118 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:50,118 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:25:50,135 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:50,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60158 deadline: 1732710410128, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:50,135 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:50,135 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:50,135 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:25:50,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-27T12:25:50,172 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T12:25:50,172 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-27T12:25:50,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] procedure2.ProcedureExecutor(1139): Stored pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees 2024-11-27T12:25:50,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-27T12:25:50,174 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T12:25:50,175 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T12:25:50,175 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=106, ppid=105, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T12:25:50,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-27T12:25:50,327 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-27T12:25:50,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068. 2024-11-27T12:25:50,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068. as already flushing 2024-11-27T12:25:50,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068. 2024-11-27T12:25:50,328 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 817ba5bcb98df62e5ed309caecfc1068, NAME => 'TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:25:50,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 817ba5bcb98df62e5ed309caecfc1068, NAME => 'TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:25:50,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 817ba5bcb98df62e5ed309caecfc1068, NAME => 'TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 817ba5bcb98df62e5ed309caecfc1068, NAME => 'TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:25:50,332 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:50,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60076 deadline: 1732710410328, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:50,332 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:50,333 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:50,333 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:25:50,357 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:50,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60158 deadline: 1732710410348, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:50,357 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:50,357 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:50,357 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:25:50,458 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=129 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/.tmp/C/8e22b3175f5e4838bea3f7f13de4e688 2024-11-27T12:25:50,465 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/.tmp/A/969e6cbf7ac643aebcd72d6b8127478e as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/A/969e6cbf7ac643aebcd72d6b8127478e 2024-11-27T12:25:50,469 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/A/969e6cbf7ac643aebcd72d6b8127478e, entries=200, sequenceid=129, filesize=38.7 K 2024-11-27T12:25:50,470 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/.tmp/B/7bbe02f709b74eb08cf8016be0748378 as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/B/7bbe02f709b74eb08cf8016be0748378 2024-11-27T12:25:50,475 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/B/7bbe02f709b74eb08cf8016be0748378, entries=150, sequenceid=129, filesize=11.8 K 2024-11-27T12:25:50,476 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/.tmp/C/8e22b3175f5e4838bea3f7f13de4e688 as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/C/8e22b3175f5e4838bea3f7f13de4e688 2024-11-27T12:25:50,482 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-27T12:25:50,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068. 2024-11-27T12:25:50,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068. as already flushing 2024-11-27T12:25:50,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068. 2024-11-27T12:25:50,482 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 817ba5bcb98df62e5ed309caecfc1068, NAME => 'TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:25:50,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 817ba5bcb98df62e5ed309caecfc1068, NAME => 'TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:25:50,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 817ba5bcb98df62e5ed309caecfc1068, NAME => 'TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 817ba5bcb98df62e5ed309caecfc1068, NAME => 'TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:25:50,488 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/C/8e22b3175f5e4838bea3f7f13de4e688, entries=150, sequenceid=129, filesize=11.8 K 2024-11-27T12:25:50,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-27T12:25:50,490 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for 817ba5bcb98df62e5ed309caecfc1068 in 601ms, sequenceid=129, compaction requested=true 2024-11-27T12:25:50,490 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 817ba5bcb98df62e5ed309caecfc1068: 2024-11-27T12:25:50,490 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T12:25:50,491 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 817ba5bcb98df62e5ed309caecfc1068:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T12:25:50,491 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T12:25:50,491 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T12:25:50,491 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101715 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T12:25:50,491 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HStore(1541): 817ba5bcb98df62e5ed309caecfc1068/A is initiating minor compaction (all files) 2024-11-27T12:25:50,491 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 817ba5bcb98df62e5ed309caecfc1068/A in TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068. 2024-11-27T12:25:50,491 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/A/40e5f80a08ee4b9993dff964912a7a7e, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/A/8889c0115b8a49eabfe31a6e3fd0c03a, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/A/969e6cbf7ac643aebcd72d6b8127478e] into tmpdir=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/.tmp, totalSize=99.3 K 2024-11-27T12:25:50,491 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068. 2024-11-27T12:25:50,492 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068. files: [hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/A/40e5f80a08ee4b9993dff964912a7a7e, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/A/8889c0115b8a49eabfe31a6e3fd0c03a, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/A/969e6cbf7ac643aebcd72d6b8127478e] 2024-11-27T12:25:50,492 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] compactions.Compactor(225): Compacting 40e5f80a08ee4b9993dff964912a7a7e, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1732710346255 2024-11-27T12:25:50,492 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 817ba5bcb98df62e5ed309caecfc1068:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T12:25:50,492 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T12:25:50,492 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 817ba5bcb98df62e5ed309caecfc1068:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T12:25:50,492 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T12:25:50,492 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36259 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T12:25:50,492 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8889c0115b8a49eabfe31a6e3fd0c03a, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1732710348532 2024-11-27T12:25:50,492 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HStore(1541): 817ba5bcb98df62e5ed309caecfc1068/B is initiating minor compaction (all files) 2024-11-27T12:25:50,493 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 817ba5bcb98df62e5ed309caecfc1068/B in TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068. 2024-11-27T12:25:50,493 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/B/52ca6d4fd2624927a2898f9b8f1a1e31, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/B/d01946b594b642509cc52b66195c09ea, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/B/7bbe02f709b74eb08cf8016be0748378] into tmpdir=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/.tmp, totalSize=35.4 K 2024-11-27T12:25:50,493 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] compactions.Compactor(225): Compacting 969e6cbf7ac643aebcd72d6b8127478e, keycount=200, bloomtype=ROW, size=38.7 K, encoding=NONE, compression=NONE, seqNum=129, earliestPutTs=1732710349728 2024-11-27T12:25:50,493 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.Compactor(225): Compacting 52ca6d4fd2624927a2898f9b8f1a1e31, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1732710346255 2024-11-27T12:25:50,493 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.Compactor(225): Compacting d01946b594b642509cc52b66195c09ea, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1732710348532 2024-11-27T12:25:50,494 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.Compactor(225): Compacting 7bbe02f709b74eb08cf8016be0748378, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=129, earliestPutTs=1732710349728 2024-11-27T12:25:50,513 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=817ba5bcb98df62e5ed309caecfc1068] 2024-11-27T12:25:50,532 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 817ba5bcb98df62e5ed309caecfc1068#B#compaction#307 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-27T12:25:50,533 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/.tmp/B/649e0c1d90e9450aa634acdfa8fe0223 is 50, key is test_row_0/B:col10/1732710349728/Put/seqid=0 2024-11-27T12:25:50,541 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411271f1edb3c126b4613ac55d5b94833c2e9_817ba5bcb98df62e5ed309caecfc1068 store=[table=TestAcidGuarantees family=A region=817ba5bcb98df62e5ed309caecfc1068] 2024-11-27T12:25:50,543 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411271f1edb3c126b4613ac55d5b94833c2e9_817ba5bcb98df62e5ed309caecfc1068, store=[table=TestAcidGuarantees family=A region=817ba5bcb98df62e5ed309caecfc1068] 2024-11-27T12:25:50,543 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411271f1edb3c126b4613ac55d5b94833c2e9_817ba5bcb98df62e5ed309caecfc1068 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=817ba5bcb98df62e5ed309caecfc1068] 2024-11-27T12:25:50,558 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742189_1365 (size=12359) 2024-11-27T12:25:50,566 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742190_1366 (size=4469) 2024-11-27T12:25:50,567 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/.tmp/B/649e0c1d90e9450aa634acdfa8fe0223 as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/B/649e0c1d90e9450aa634acdfa8fe0223 2024-11-27T12:25:50,573 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 817ba5bcb98df62e5ed309caecfc1068/B of 817ba5bcb98df62e5ed309caecfc1068 into 649e0c1d90e9450aa634acdfa8fe0223(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T12:25:50,573 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 817ba5bcb98df62e5ed309caecfc1068: 2024-11-27T12:25:50,573 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., storeName=817ba5bcb98df62e5ed309caecfc1068/B, priority=13, startTime=1732710350491; duration=0sec 2024-11-27T12:25:50,573 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T12:25:50,573 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 817ba5bcb98df62e5ed309caecfc1068:B 2024-11-27T12:25:50,573 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-27T12:25:50,574 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-27T12:25:50,574 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-27T12:25:50,574 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068. because compaction request was cancelled 2024-11-27T12:25:50,574 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 817ba5bcb98df62e5ed309caecfc1068:C 2024-11-27T12:25:50,581 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 817ba5bcb98df62e5ed309caecfc1068#A#compaction#306 average throughput is 0.36 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T12:25:50,582 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/.tmp/A/520e75c14a9e4e08b88eecde24c10354 is 175, key is test_row_0/A:col10/1732710349728/Put/seqid=0 2024-11-27T12:25:50,594 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742191_1367 (size=31313) 2024-11-27T12:25:50,602 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/.tmp/A/520e75c14a9e4e08b88eecde24c10354 as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/A/520e75c14a9e4e08b88eecde24c10354 2024-11-27T12:25:50,608 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 817ba5bcb98df62e5ed309caecfc1068/A of 817ba5bcb98df62e5ed309caecfc1068 into 520e75c14a9e4e08b88eecde24c10354(size=30.6 K), total size for store is 30.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T12:25:50,608 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 817ba5bcb98df62e5ed309caecfc1068: 2024-11-27T12:25:50,608 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., storeName=817ba5bcb98df62e5ed309caecfc1068/A, priority=13, startTime=1732710350490; duration=0sec 2024-11-27T12:25:50,608 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T12:25:50,608 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 817ba5bcb98df62e5ed309caecfc1068:A 2024-11-27T12:25:50,637 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-27T12:25:50,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068. 2024-11-27T12:25:50,638 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2902): Flushing 817ba5bcb98df62e5ed309caecfc1068 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-27T12:25:50,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(8855): Flush requested on 817ba5bcb98df62e5ed309caecfc1068 2024-11-27T12:25:50,649 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068. as already flushing 2024-11-27T12:25:50,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411278aae8f7a3cc14bd69ea2c3e0ef847563_817ba5bcb98df62e5ed309caecfc1068 is 50, key is test_row_0/A:col10/1732710349990/Put/seqid=0 2024-11-27T12:25:50,668 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742192_1368 (size=12304) 2024-11-27T12:25:50,703 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:50,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60076 deadline: 1732710410695, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:50,703 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:50,703 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:50,703 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:50,703 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:25:50,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60158 deadline: 1732710410697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:50,704 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:50,704 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:50,704 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:25:50,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-27T12:25:50,811 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:50,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60076 deadline: 1732710410807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:50,811 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:50,811 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:50,812 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:50,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60158 deadline: 1732710410807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:50,812 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:25:50,812 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:50,812 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:50,812 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:25:51,027 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:51,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60158 deadline: 1732710411018, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:51,028 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:51,028 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:51,028 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:51,028 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:25:51,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60076 deadline: 1732710411018, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:51,028 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:51,028 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:51,028 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:25:51,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:25:51,072 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411278aae8f7a3cc14bd69ea2c3e0ef847563_817ba5bcb98df62e5ed309caecfc1068 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411278aae8f7a3cc14bd69ea2c3e0ef847563_817ba5bcb98df62e5ed309caecfc1068 2024-11-27T12:25:51,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/.tmp/A/e03e11f7b8334c7ba8c976fa7a6db5cf, store: [table=TestAcidGuarantees family=A region=817ba5bcb98df62e5ed309caecfc1068] 2024-11-27T12:25:51,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/.tmp/A/e03e11f7b8334c7ba8c976fa7a6db5cf is 175, key is test_row_0/A:col10/1732710349990/Put/seqid=0 2024-11-27T12:25:51,080 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742193_1369 (size=31105) 2024-11-27T12:25:51,197 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-27T12:25:51,197 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-27T12:25:51,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-27T12:25:51,346 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:51,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60158 deadline: 1732710411338, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:51,346 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:51,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60076 deadline: 1732710411338, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:51,346 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:51,346 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:51,346 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:25:51,347 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:51,347 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:51,347 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:25:51,480 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=156, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/.tmp/A/e03e11f7b8334c7ba8c976fa7a6db5cf 2024-11-27T12:25:51,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/.tmp/B/caf4d68a8a974141b6385cc56e74324b is 50, key is test_row_0/B:col10/1732710349990/Put/seqid=0 2024-11-27T12:25:51,505 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742194_1370 (size=12151) 2024-11-27T12:25:51,505 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/.tmp/B/caf4d68a8a974141b6385cc56e74324b 2024-11-27T12:25:51,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/.tmp/C/ddfcb19b8e8a4e9f95f465cf8750535d is 50, key is test_row_0/C:col10/1732710349990/Put/seqid=0 2024-11-27T12:25:51,522 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742195_1371 (size=12151) 2024-11-27T12:25:51,523 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/.tmp/C/ddfcb19b8e8a4e9f95f465cf8750535d 2024-11-27T12:25:51,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/.tmp/A/e03e11f7b8334c7ba8c976fa7a6db5cf as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/A/e03e11f7b8334c7ba8c976fa7a6db5cf 2024-11-27T12:25:51,532 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/A/e03e11f7b8334c7ba8c976fa7a6db5cf, entries=150, sequenceid=156, filesize=30.4 K 2024-11-27T12:25:51,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/.tmp/B/caf4d68a8a974141b6385cc56e74324b as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/B/caf4d68a8a974141b6385cc56e74324b 2024-11-27T12:25:51,538 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/B/caf4d68a8a974141b6385cc56e74324b, entries=150, sequenceid=156, filesize=11.9 K 2024-11-27T12:25:51,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/.tmp/C/ddfcb19b8e8a4e9f95f465cf8750535d as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/C/ddfcb19b8e8a4e9f95f465cf8750535d 2024-11-27T12:25:51,542 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/C/ddfcb19b8e8a4e9f95f465cf8750535d, entries=150, sequenceid=156, filesize=11.9 K 2024-11-27T12:25:51,544 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for 817ba5bcb98df62e5ed309caecfc1068 in 907ms, sequenceid=156, compaction requested=true 2024-11-27T12:25:51,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2603): Flush status journal for 817ba5bcb98df62e5ed309caecfc1068: 2024-11-27T12:25:51,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068. 2024-11-27T12:25:51,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=106 2024-11-27T12:25:51,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.HMaster(4169): Remote procedure done, pid=106 2024-11-27T12:25:51,547 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=106, resume processing ppid=105 2024-11-27T12:25:51,548 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=106, ppid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3710 sec 2024-11-27T12:25:51,549 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees in 1.3760 sec 2024-11-27T12:25:51,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(8855): Flush requested on 817ba5bcb98df62e5ed309caecfc1068 2024-11-27T12:25:51,866 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T12:25:51,866 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 817ba5bcb98df62e5ed309caecfc1068 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-27T12:25:51,874 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411278411adc70526415d8fc3f16e08e5c20b_817ba5bcb98df62e5ed309caecfc1068 is 50, key is test_row_0/A:col10/1732710350696/Put/seqid=0 2024-11-27T12:25:51,893 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742196_1372 (size=14794) 2024-11-27T12:25:51,939 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:51,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60158 deadline: 1732710411934, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:51,940 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:51,940 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:51,940 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:25:51,942 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:51,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60076 deadline: 1732710411940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:51,942 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:51,942 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:51,942 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:25:52,056 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:52,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60076 deadline: 1732710412047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:52,056 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:52,056 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:52,056 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:52,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60158 deadline: 1732710412048, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:52,056 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:25:52,056 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:52,056 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:52,057 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:25:52,275 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:52,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60076 deadline: 1732710412268, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:52,276 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:52,276 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:52,276 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:52,276 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:25:52,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60158 deadline: 1732710412268, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:52,276 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:52,276 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:52,276 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:25:52,294 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:25:52,298 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411278411adc70526415d8fc3f16e08e5c20b_817ba5bcb98df62e5ed309caecfc1068 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411278411adc70526415d8fc3f16e08e5c20b_817ba5bcb98df62e5ed309caecfc1068 2024-11-27T12:25:52,299 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/.tmp/A/14094e0bbda94990808621bc8ac2143a, store: [table=TestAcidGuarantees family=A region=817ba5bcb98df62e5ed309caecfc1068] 2024-11-27T12:25:52,299 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/.tmp/A/14094e0bbda94990808621bc8ac2143a is 175, key is test_row_0/A:col10/1732710350696/Put/seqid=0 2024-11-27T12:25:52,308 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742197_1373 (size=39749) 2024-11-27T12:25:52,309 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=171, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/.tmp/A/14094e0bbda94990808621bc8ac2143a 2024-11-27T12:25:52,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-27T12:25:52,320 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T12:25:52,321 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-27T12:25:52,322 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/.tmp/B/2059facbbd2e477b93bb7521e62bb7f1 is 50, key is test_row_0/B:col10/1732710350696/Put/seqid=0 2024-11-27T12:25:52,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] procedure2.ProcedureExecutor(1139): Stored pid=107, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees 2024-11-27T12:25:52,323 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=107, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T12:25:52,325 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=107, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T12:25:52,325 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=108, ppid=107, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T12:25:52,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-27T12:25:52,329 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742198_1374 (size=12151) 2024-11-27T12:25:52,330 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=171 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/.tmp/B/2059facbbd2e477b93bb7521e62bb7f1 2024-11-27T12:25:52,346 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/.tmp/C/141608c0d11a41ba99a9142cd9b4f431 is 50, key is test_row_0/C:col10/1732710350696/Put/seqid=0 2024-11-27T12:25:52,350 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742199_1375 (size=12151) 2024-11-27T12:25:52,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-27T12:25:52,478 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-27T12:25:52,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068. 2024-11-27T12:25:52,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068. as already flushing 2024-11-27T12:25:52,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068. 2024-11-27T12:25:52,478 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 817ba5bcb98df62e5ed309caecfc1068, NAME => 'TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:25:52,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 817ba5bcb98df62e5ed309caecfc1068, NAME => 'TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:25:52,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 817ba5bcb98df62e5ed309caecfc1068, NAME => 'TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 817ba5bcb98df62e5ed309caecfc1068, NAME => 'TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:25:52,582 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:52,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60076 deadline: 1732710412578, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:52,583 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:52,583 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:52,583 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:25:52,595 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:52,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60158 deadline: 1732710412588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:52,595 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:52,595 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:52,595 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:25:52,632 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-27T12:25:52,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068. 2024-11-27T12:25:52,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068. as already flushing 2024-11-27T12:25:52,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068. 2024-11-27T12:25:52,632 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 817ba5bcb98df62e5ed309caecfc1068, NAME => 'TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:25:52,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 817ba5bcb98df62e5ed309caecfc1068, NAME => 'TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:25:52,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 817ba5bcb98df62e5ed309caecfc1068, NAME => 'TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 817ba5bcb98df62e5ed309caecfc1068, NAME => 'TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:25:52,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-27T12:25:52,665 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:52,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60150 deadline: 1732710412658, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:52,666 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 1ba5af43fd99,36251,1732710241631 for 'test_row_0' in 817ba5bcb98df62e5ed309caecfc1068 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8223 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T12:25:52,666 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:52,666 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:52,666 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:25:52,678 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:52,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60108 deadline: 1732710412668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:52,678 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 1ba5af43fd99,36251,1732710241631 for 'test_row_0' in 817ba5bcb98df62e5ed309caecfc1068 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8232 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T12:25:52,679 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:52,679 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:52,679 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:25:52,686 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:52,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60134 deadline: 1732710412677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:52,686 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 1ba5af43fd99,36251,1732710241631 for 'test_row_0' in 817ba5bcb98df62e5ed309caecfc1068 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8243 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T12:25:52,687 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:52,687 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:52,687 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:25:52,750 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=171 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/.tmp/C/141608c0d11a41ba99a9142cd9b4f431 2024-11-27T12:25:52,754 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/.tmp/A/14094e0bbda94990808621bc8ac2143a as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/A/14094e0bbda94990808621bc8ac2143a 2024-11-27T12:25:52,758 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/A/14094e0bbda94990808621bc8ac2143a, entries=200, sequenceid=171, filesize=38.8 K 2024-11-27T12:25:52,759 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/.tmp/B/2059facbbd2e477b93bb7521e62bb7f1 as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/B/2059facbbd2e477b93bb7521e62bb7f1 2024-11-27T12:25:52,762 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/B/2059facbbd2e477b93bb7521e62bb7f1, entries=150, sequenceid=171, filesize=11.9 K 2024-11-27T12:25:52,763 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/.tmp/C/141608c0d11a41ba99a9142cd9b4f431 as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/C/141608c0d11a41ba99a9142cd9b4f431 2024-11-27T12:25:52,767 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/C/141608c0d11a41ba99a9142cd9b4f431, entries=150, sequenceid=171, filesize=11.9 K 2024-11-27T12:25:52,767 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for 817ba5bcb98df62e5ed309caecfc1068 in 901ms, sequenceid=171, compaction requested=true 2024-11-27T12:25:52,768 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 817ba5bcb98df62e5ed309caecfc1068: 2024-11-27T12:25:52,768 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 817ba5bcb98df62e5ed309caecfc1068:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T12:25:52,768 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T12:25:52,768 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T12:25:52,768 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 817ba5bcb98df62e5ed309caecfc1068:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T12:25:52,768 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T12:25:52,768 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T12:25:52,768 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 817ba5bcb98df62e5ed309caecfc1068:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T12:25:52,768 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T12:25:52,769 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36661 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T12:25:52,769 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102167 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T12:25:52,769 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HStore(1541): 817ba5bcb98df62e5ed309caecfc1068/B is initiating minor compaction (all files) 2024-11-27T12:25:52,769 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HStore(1541): 817ba5bcb98df62e5ed309caecfc1068/A is initiating minor compaction (all files) 2024-11-27T12:25:52,769 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 817ba5bcb98df62e5ed309caecfc1068/A in TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068. 2024-11-27T12:25:52,769 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 817ba5bcb98df62e5ed309caecfc1068/B in TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068. 2024-11-27T12:25:52,769 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/A/520e75c14a9e4e08b88eecde24c10354, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/A/e03e11f7b8334c7ba8c976fa7a6db5cf, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/A/14094e0bbda94990808621bc8ac2143a] into tmpdir=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/.tmp, totalSize=99.8 K 2024-11-27T12:25:52,769 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/B/649e0c1d90e9450aa634acdfa8fe0223, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/B/caf4d68a8a974141b6385cc56e74324b, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/B/2059facbbd2e477b93bb7521e62bb7f1] into tmpdir=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/.tmp, totalSize=35.8 K 2024-11-27T12:25:52,769 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068. 2024-11-27T12:25:52,769 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068. files: [hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/A/520e75c14a9e4e08b88eecde24c10354, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/A/e03e11f7b8334c7ba8c976fa7a6db5cf, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/A/14094e0bbda94990808621bc8ac2143a] 2024-11-27T12:25:52,769 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.Compactor(225): Compacting 649e0c1d90e9450aa634acdfa8fe0223, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=129, earliestPutTs=1732710349728 2024-11-27T12:25:52,769 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] compactions.Compactor(225): Compacting 520e75c14a9e4e08b88eecde24c10354, keycount=150, bloomtype=ROW, size=30.6 K, encoding=NONE, compression=NONE, seqNum=129, earliestPutTs=1732710349728 2024-11-27T12:25:52,769 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.Compactor(225): Compacting caf4d68a8a974141b6385cc56e74324b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1732710349947 2024-11-27T12:25:52,769 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] compactions.Compactor(225): Compacting e03e11f7b8334c7ba8c976fa7a6db5cf, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1732710349947 2024-11-27T12:25:52,769 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.Compactor(225): Compacting 2059facbbd2e477b93bb7521e62bb7f1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1732710350673 2024-11-27T12:25:52,770 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] compactions.Compactor(225): Compacting 14094e0bbda94990808621bc8ac2143a, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1732710350673 2024-11-27T12:25:52,778 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=817ba5bcb98df62e5ed309caecfc1068] 2024-11-27T12:25:52,784 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 817ba5bcb98df62e5ed309caecfc1068#B#compaction#315 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-27T12:25:52,784 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/.tmp/B/4cdd3531e02243b682fde13b024a3e23 is 50, key is test_row_0/B:col10/1732710350696/Put/seqid=0 2024-11-27T12:25:52,784 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-27T12:25:52,785 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024112706a653f164b14346bfff6fbb8062a5c4_817ba5bcb98df62e5ed309caecfc1068 store=[table=TestAcidGuarantees family=A region=817ba5bcb98df62e5ed309caecfc1068] 2024-11-27T12:25:52,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068. 2024-11-27T12:25:52,785 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2902): Flushing 817ba5bcb98df62e5ed309caecfc1068 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-27T12:25:52,787 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024112706a653f164b14346bfff6fbb8062a5c4_817ba5bcb98df62e5ed309caecfc1068, store=[table=TestAcidGuarantees family=A region=817ba5bcb98df62e5ed309caecfc1068] 2024-11-27T12:25:52,787 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112706a653f164b14346bfff6fbb8062a5c4_817ba5bcb98df62e5ed309caecfc1068 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=817ba5bcb98df62e5ed309caecfc1068] 2024-11-27T12:25:52,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411274301c71aa7f94da98cf3e933ba9ff650_817ba5bcb98df62e5ed309caecfc1068 is 50, key is test_row_0/A:col10/1732710351933/Put/seqid=0 2024-11-27T12:25:52,808 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742200_1376 (size=4469) 2024-11-27T12:25:52,811 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 817ba5bcb98df62e5ed309caecfc1068#A#compaction#314 average throughput is 0.74 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T12:25:52,811 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/.tmp/A/4b2c4ec6b8b942ccb7d88937d3892b7b is 175, key is test_row_0/A:col10/1732710350696/Put/seqid=0 2024-11-27T12:25:52,821 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742201_1377 (size=12561) 2024-11-27T12:25:52,830 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742202_1378 (size=12304) 2024-11-27T12:25:52,839 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742203_1379 (size=31515) 2024-11-27T12:25:52,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-27T12:25:53,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(8855): Flush requested on 817ba5bcb98df62e5ed309caecfc1068 2024-11-27T12:25:53,094 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068. as already flushing 2024-11-27T12:25:53,156 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:53,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60158 deadline: 1732710413145, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:53,157 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:53,157 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:53,157 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:25:53,160 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:53,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60076 deadline: 1732710413147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:53,160 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:53,160 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:53,160 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:25:53,227 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/.tmp/B/4cdd3531e02243b682fde13b024a3e23 as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/B/4cdd3531e02243b682fde13b024a3e23 2024-11-27T12:25:53,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:25:53,237 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 817ba5bcb98df62e5ed309caecfc1068/B of 817ba5bcb98df62e5ed309caecfc1068 into 4cdd3531e02243b682fde13b024a3e23(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T12:25:53,237 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 817ba5bcb98df62e5ed309caecfc1068: 2024-11-27T12:25:53,237 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., storeName=817ba5bcb98df62e5ed309caecfc1068/B, priority=13, startTime=1732710352768; duration=0sec 2024-11-27T12:25:53,237 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T12:25:53,237 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 817ba5bcb98df62e5ed309caecfc1068:B 2024-11-27T12:25:53,237 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-27T12:25:53,239 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411274301c71aa7f94da98cf3e933ba9ff650_817ba5bcb98df62e5ed309caecfc1068 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411274301c71aa7f94da98cf3e933ba9ff650_817ba5bcb98df62e5ed309caecfc1068 2024-11-27T12:25:53,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/.tmp/A/b010378453de41a59802977b340a1c84, store: [table=TestAcidGuarantees family=A region=817ba5bcb98df62e5ed309caecfc1068] 2024-11-27T12:25:53,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/.tmp/A/b010378453de41a59802977b340a1c84 is 175, key is test_row_0/A:col10/1732710351933/Put/seqid=0 2024-11-27T12:25:53,241 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48594 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-27T12:25:53,241 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HStore(1541): 817ba5bcb98df62e5ed309caecfc1068/C is initiating minor compaction (all files) 2024-11-27T12:25:53,241 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 817ba5bcb98df62e5ed309caecfc1068/C in TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068. 2024-11-27T12:25:53,241 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/C/06b2c9f79fd940c9965d97cf9b259ab4, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/C/8e22b3175f5e4838bea3f7f13de4e688, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/C/ddfcb19b8e8a4e9f95f465cf8750535d, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/C/141608c0d11a41ba99a9142cd9b4f431] into tmpdir=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/.tmp, totalSize=47.5 K 2024-11-27T12:25:53,242 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.Compactor(225): Compacting 06b2c9f79fd940c9965d97cf9b259ab4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1732710348532 2024-11-27T12:25:53,243 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.Compactor(225): Compacting 8e22b3175f5e4838bea3f7f13de4e688, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=129, earliestPutTs=1732710349728 2024-11-27T12:25:53,244 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.Compactor(225): Compacting ddfcb19b8e8a4e9f95f465cf8750535d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1732710349947 2024-11-27T12:25:53,245 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.Compactor(225): Compacting 141608c0d11a41ba99a9142cd9b4f431, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1732710350673 2024-11-27T12:25:53,259 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/.tmp/A/4b2c4ec6b8b942ccb7d88937d3892b7b as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/A/4b2c4ec6b8b942ccb7d88937d3892b7b 2024-11-27T12:25:53,268 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 817ba5bcb98df62e5ed309caecfc1068/A of 817ba5bcb98df62e5ed309caecfc1068 into 4b2c4ec6b8b942ccb7d88937d3892b7b(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T12:25:53,268 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 817ba5bcb98df62e5ed309caecfc1068: 2024-11-27T12:25:53,268 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., storeName=817ba5bcb98df62e5ed309caecfc1068/A, priority=13, startTime=1732710352768; duration=0sec 2024-11-27T12:25:53,268 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T12:25:53,268 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 817ba5bcb98df62e5ed309caecfc1068:A 2024-11-27T12:25:53,278 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:53,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60158 deadline: 1732710413269, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:53,278 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:53,278 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:53,278 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:25:53,279 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:53,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60076 deadline: 1732710413271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:53,279 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:53,279 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:53,279 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:25:53,283 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 817ba5bcb98df62e5ed309caecfc1068#C#compaction#317 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T12:25:53,284 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/.tmp/C/0cef1e2389394b66a588690dd0d4ce30 is 50, key is test_row_0/C:col10/1732710350696/Put/seqid=0 2024-11-27T12:25:53,285 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742204_1380 (size=31105) 2024-11-27T12:25:53,286 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=192, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/.tmp/A/b010378453de41a59802977b340a1c84 2024-11-27T12:25:53,304 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/.tmp/B/1315c53304f34655859df25e0a66925e is 50, key is test_row_0/B:col10/1732710351933/Put/seqid=0 2024-11-27T12:25:53,306 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742205_1381 (size=12527) 2024-11-27T12:25:53,316 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742206_1382 (size=12151) 2024-11-27T12:25:53,317 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=192 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/.tmp/B/1315c53304f34655859df25e0a66925e 2024-11-27T12:25:53,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/.tmp/C/4cd99bd20d0244168a58fd5af97647f0 is 50, key is test_row_0/C:col10/1732710351933/Put/seqid=0 2024-11-27T12:25:53,337 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742207_1383 (size=12151) 2024-11-27T12:25:53,338 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=192 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/.tmp/C/4cd99bd20d0244168a58fd5af97647f0 2024-11-27T12:25:53,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/.tmp/A/b010378453de41a59802977b340a1c84 as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/A/b010378453de41a59802977b340a1c84 2024-11-27T12:25:53,349 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/A/b010378453de41a59802977b340a1c84, entries=150, sequenceid=192, filesize=30.4 K 2024-11-27T12:25:53,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/.tmp/B/1315c53304f34655859df25e0a66925e as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/B/1315c53304f34655859df25e0a66925e 2024-11-27T12:25:53,355 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/B/1315c53304f34655859df25e0a66925e, entries=150, sequenceid=192, filesize=11.9 K 2024-11-27T12:25:53,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/.tmp/C/4cd99bd20d0244168a58fd5af97647f0 as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/C/4cd99bd20d0244168a58fd5af97647f0 2024-11-27T12:25:53,361 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/C/4cd99bd20d0244168a58fd5af97647f0, entries=150, sequenceid=192, filesize=11.9 K 2024-11-27T12:25:53,363 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=80.51 KB/82440 for 817ba5bcb98df62e5ed309caecfc1068 in 578ms, sequenceid=192, compaction requested=false 2024-11-27T12:25:53,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2603): Flush status journal for 817ba5bcb98df62e5ed309caecfc1068: 2024-11-27T12:25:53,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068. 2024-11-27T12:25:53,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=108 2024-11-27T12:25:53,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.HMaster(4169): Remote procedure done, pid=108 2024-11-27T12:25:53,366 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=108, resume processing ppid=107 2024-11-27T12:25:53,366 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=108, ppid=107, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0400 sec 2024-11-27T12:25:53,369 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=107, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees in 1.0460 sec 2024-11-27T12:25:53,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-27T12:25:53,459 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T12:25:53,459 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-27T12:25:53,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] procedure2.ProcedureExecutor(1139): Stored pid=109, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees 2024-11-27T12:25:53,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-27T12:25:53,461 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=109, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T12:25:53,461 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=109, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T12:25:53,461 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=110, ppid=109, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T12:25:53,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(8855): Flush requested on 817ba5bcb98df62e5ed309caecfc1068 2024-11-27T12:25:53,492 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T12:25:53,492 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 817ba5bcb98df62e5ed309caecfc1068 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-27T12:25:53,499 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127e80b5daef4e94291b6fb031c0cabd27b_817ba5bcb98df62e5ed309caecfc1068 is 50, key is test_row_0/A:col10/1732710353115/Put/seqid=0 2024-11-27T12:25:53,503 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742208_1384 (size=17284) 2024-11-27T12:25:53,504 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:25:53,507 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127e80b5daef4e94291b6fb031c0cabd27b_817ba5bcb98df62e5ed309caecfc1068 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127e80b5daef4e94291b6fb031c0cabd27b_817ba5bcb98df62e5ed309caecfc1068 2024-11-27T12:25:53,508 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/.tmp/A/512b281878f04be1bb55668a4602e36e, store: [table=TestAcidGuarantees family=A region=817ba5bcb98df62e5ed309caecfc1068] 2024-11-27T12:25:53,508 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/.tmp/A/512b281878f04be1bb55668a4602e36e is 175, key is test_row_0/A:col10/1732710353115/Put/seqid=0 2024-11-27T12:25:53,516 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742209_1385 (size=48389) 2024-11-27T12:25:53,516 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=211, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/.tmp/A/512b281878f04be1bb55668a4602e36e 2024-11-27T12:25:53,522 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/.tmp/B/4c9cae6bfb014bb0a23800e3f126bf36 is 50, key is test_row_0/B:col10/1732710353115/Put/seqid=0 2024-11-27T12:25:53,525 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742210_1386 (size=12151) 2024-11-27T12:25:53,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-27T12:25:53,591 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:53,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60158 deadline: 1732710413584, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:53,591 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:53,591 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:53,592 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:25:53,597 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:53,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60076 deadline: 1732710413589, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:53,598 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:53,598 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:53,598 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:25:53,613 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-27T12:25:53,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068. 2024-11-27T12:25:53,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068. as already flushing 2024-11-27T12:25:53,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068. 2024-11-27T12:25:53,613 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 817ba5bcb98df62e5ed309caecfc1068, NAME => 'TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:25:53,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 817ba5bcb98df62e5ed309caecfc1068, NAME => 'TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:25:53,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 817ba5bcb98df62e5ed309caecfc1068, NAME => 'TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 817ba5bcb98df62e5ed309caecfc1068, NAME => 'TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:25:53,702 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:53,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60158 deadline: 1732710413697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:53,702 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:53,702 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:53,702 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:25:53,709 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/.tmp/C/0cef1e2389394b66a588690dd0d4ce30 as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/C/0cef1e2389394b66a588690dd0d4ce30 2024-11-27T12:25:53,713 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:53,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60076 deadline: 1732710413708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:53,713 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 817ba5bcb98df62e5ed309caecfc1068/C of 817ba5bcb98df62e5ed309caecfc1068 into 0cef1e2389394b66a588690dd0d4ce30(size=12.2 K), total size for store is 24.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T12:25:53,713 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 817ba5bcb98df62e5ed309caecfc1068: 2024-11-27T12:25:53,713 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., storeName=817ba5bcb98df62e5ed309caecfc1068/C, priority=12, startTime=1732710352768; duration=0sec 2024-11-27T12:25:53,713 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:53,713 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T12:25:53,714 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:53,714 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 817ba5bcb98df62e5ed309caecfc1068:C 2024-11-27T12:25:53,714 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:25:53,766 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-27T12:25:53,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068. 2024-11-27T12:25:53,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068. as already flushing 2024-11-27T12:25:53,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068. 2024-11-27T12:25:53,766 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 817ba5bcb98df62e5ed309caecfc1068, NAME => 'TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:25:53,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 817ba5bcb98df62e5ed309caecfc1068, NAME => 'TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:25:53,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 817ba5bcb98df62e5ed309caecfc1068, NAME => 'TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 817ba5bcb98df62e5ed309caecfc1068, NAME => 'TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:25:53,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-27T12:25:53,912 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:53,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60158 deadline: 1732710413909, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:53,913 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:53,913 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:53,913 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:25:53,919 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-27T12:25:53,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068. 2024-11-27T12:25:53,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068. as already flushing 2024-11-27T12:25:53,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068. 2024-11-27T12:25:53,919 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 817ba5bcb98df62e5ed309caecfc1068, NAME => 'TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:25:53,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 817ba5bcb98df62e5ed309caecfc1068, NAME => 'TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:25:53,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 817ba5bcb98df62e5ed309caecfc1068, NAME => 'TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 817ba5bcb98df62e5ed309caecfc1068, NAME => 'TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:25:53,926 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:53,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60076 deadline: 1732710413918, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:53,926 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:53,926 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:53,926 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:25:53,926 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=211 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/.tmp/B/4c9cae6bfb014bb0a23800e3f126bf36 2024-11-27T12:25:53,946 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/.tmp/C/5682906c1d364e31871391035db45e55 is 50, key is test_row_0/C:col10/1732710353115/Put/seqid=0 2024-11-27T12:25:53,955 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742211_1387 (size=12151) 2024-11-27T12:25:53,955 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=211 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/.tmp/C/5682906c1d364e31871391035db45e55 2024-11-27T12:25:53,959 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/.tmp/A/512b281878f04be1bb55668a4602e36e as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/A/512b281878f04be1bb55668a4602e36e 2024-11-27T12:25:53,963 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/A/512b281878f04be1bb55668a4602e36e, entries=250, sequenceid=211, filesize=47.3 K 2024-11-27T12:25:53,964 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/.tmp/B/4c9cae6bfb014bb0a23800e3f126bf36 as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/B/4c9cae6bfb014bb0a23800e3f126bf36 2024-11-27T12:25:53,970 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/B/4c9cae6bfb014bb0a23800e3f126bf36, entries=150, sequenceid=211, filesize=11.9 K 2024-11-27T12:25:53,971 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/.tmp/C/5682906c1d364e31871391035db45e55 as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/C/5682906c1d364e31871391035db45e55 2024-11-27T12:25:53,975 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/C/5682906c1d364e31871391035db45e55, entries=150, sequenceid=211, filesize=11.9 K 2024-11-27T12:25:53,976 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=107.34 KB/109920 for 817ba5bcb98df62e5ed309caecfc1068 in 484ms, sequenceid=211, compaction requested=true 2024-11-27T12:25:53,976 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 817ba5bcb98df62e5ed309caecfc1068: 2024-11-27T12:25:53,976 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 817ba5bcb98df62e5ed309caecfc1068:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T12:25:53,977 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T12:25:53,977 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T12:25:53,977 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T12:25:53,977 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 817ba5bcb98df62e5ed309caecfc1068:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T12:25:53,977 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T12:25:53,978 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 111009 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T12:25:53,978 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T12:25:53,978 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HStore(1541): 817ba5bcb98df62e5ed309caecfc1068/B is initiating minor compaction (all files) 2024-11-27T12:25:53,978 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HStore(1541): 817ba5bcb98df62e5ed309caecfc1068/A is initiating minor compaction (all files) 2024-11-27T12:25:53,978 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 817ba5bcb98df62e5ed309caecfc1068/A in TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068. 2024-11-27T12:25:53,978 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 817ba5bcb98df62e5ed309caecfc1068/B in TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068. 2024-11-27T12:25:53,978 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/A/4b2c4ec6b8b942ccb7d88937d3892b7b, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/A/b010378453de41a59802977b340a1c84, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/A/512b281878f04be1bb55668a4602e36e] into tmpdir=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/.tmp, totalSize=108.4 K 2024-11-27T12:25:53,978 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/B/4cdd3531e02243b682fde13b024a3e23, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/B/1315c53304f34655859df25e0a66925e, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/B/4c9cae6bfb014bb0a23800e3f126bf36] into tmpdir=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/.tmp, totalSize=36.0 K 2024-11-27T12:25:53,978 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068. 2024-11-27T12:25:53,978 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 817ba5bcb98df62e5ed309caecfc1068:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T12:25:53,978 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T12:25:53,978 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068. files: [hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/A/4b2c4ec6b8b942ccb7d88937d3892b7b, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/A/b010378453de41a59802977b340a1c84, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/A/512b281878f04be1bb55668a4602e36e] 2024-11-27T12:25:53,979 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.Compactor(225): Compacting 4cdd3531e02243b682fde13b024a3e23, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1732710350673 2024-11-27T12:25:53,979 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4b2c4ec6b8b942ccb7d88937d3892b7b, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1732710350673 2024-11-27T12:25:53,979 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] compactions.Compactor(225): Compacting b010378453de41a59802977b340a1c84, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=192, earliestPutTs=1732710351915 2024-11-27T12:25:53,979 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.Compactor(225): Compacting 1315c53304f34655859df25e0a66925e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=192, earliestPutTs=1732710351915 2024-11-27T12:25:53,979 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.Compactor(225): Compacting 4c9cae6bfb014bb0a23800e3f126bf36, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1732710353115 2024-11-27T12:25:53,979 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] compactions.Compactor(225): Compacting 512b281878f04be1bb55668a4602e36e, keycount=250, bloomtype=ROW, size=47.3 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1732710353115 2024-11-27T12:25:53,993 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=817ba5bcb98df62e5ed309caecfc1068] 2024-11-27T12:25:53,998 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 817ba5bcb98df62e5ed309caecfc1068#B#compaction#324 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-27T12:25:53,998 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/.tmp/B/6a5d768be57c4a91b822d6c4173755b9 is 50, key is test_row_0/B:col10/1732710353115/Put/seqid=0 2024-11-27T12:25:54,009 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241127d2a68c7a91c144a9bae40c2e5f025593_817ba5bcb98df62e5ed309caecfc1068 store=[table=TestAcidGuarantees family=A region=817ba5bcb98df62e5ed309caecfc1068] 2024-11-27T12:25:54,011 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241127d2a68c7a91c144a9bae40c2e5f025593_817ba5bcb98df62e5ed309caecfc1068, store=[table=TestAcidGuarantees family=A region=817ba5bcb98df62e5ed309caecfc1068] 2024-11-27T12:25:54,011 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127d2a68c7a91c144a9bae40c2e5f025593_817ba5bcb98df62e5ed309caecfc1068 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=817ba5bcb98df62e5ed309caecfc1068] 2024-11-27T12:25:54,034 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742212_1388 (size=12663) 2024-11-27T12:25:54,041 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742213_1389 (size=4469) 2024-11-27T12:25:54,042 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 817ba5bcb98df62e5ed309caecfc1068#A#compaction#323 average throughput is 0.50 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T12:25:54,043 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/.tmp/A/c27c9d3696ea427a9422cb55df4e1c90 is 175, key is test_row_0/A:col10/1732710353115/Put/seqid=0 2024-11-27T12:25:54,059 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742214_1390 (size=31617) 2024-11-27T12:25:54,071 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-27T12:25:54,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068. 2024-11-27T12:25:54,072 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2902): Flushing 817ba5bcb98df62e5ed309caecfc1068 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-11-27T12:25:54,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411277137b0c26e644c0daf6ae44dd6b9c7e7_817ba5bcb98df62e5ed309caecfc1068 is 50, key is test_row_0/A:col10/1732710353587/Put/seqid=0 2024-11-27T12:25:54,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-27T12:25:54,097 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742215_1391 (size=12304) 2024-11-27T12:25:54,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:25:54,107 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411277137b0c26e644c0daf6ae44dd6b9c7e7_817ba5bcb98df62e5ed309caecfc1068 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411277137b0c26e644c0daf6ae44dd6b9c7e7_817ba5bcb98df62e5ed309caecfc1068 2024-11-27T12:25:54,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/.tmp/A/04342a17b9bc486a965170f4df8af49a, store: [table=TestAcidGuarantees family=A region=817ba5bcb98df62e5ed309caecfc1068] 2024-11-27T12:25:54,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/.tmp/A/04342a17b9bc486a965170f4df8af49a is 175, key is test_row_0/A:col10/1732710353587/Put/seqid=0 2024-11-27T12:25:54,135 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742216_1392 (size=31105) 2024-11-27T12:25:54,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(8855): Flush requested on 817ba5bcb98df62e5ed309caecfc1068 2024-11-27T12:25:54,223 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068. as already flushing 2024-11-27T12:25:54,311 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:54,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60076 deadline: 1732710414298, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:54,312 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:54,312 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:54,312 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:25:54,317 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:54,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60158 deadline: 1732710414303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:54,318 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:54,318 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:54,318 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:25:54,430 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:54,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60076 deadline: 1732710414418, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:54,430 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:54,430 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:54,430 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:25:54,434 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:54,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60158 deadline: 1732710414429, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:54,435 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:54,435 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:54,435 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:25:54,441 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/.tmp/B/6a5d768be57c4a91b822d6c4173755b9 as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/B/6a5d768be57c4a91b822d6c4173755b9 2024-11-27T12:25:54,457 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 817ba5bcb98df62e5ed309caecfc1068/B of 817ba5bcb98df62e5ed309caecfc1068 into 6a5d768be57c4a91b822d6c4173755b9(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T12:25:54,457 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 817ba5bcb98df62e5ed309caecfc1068: 2024-11-27T12:25:54,458 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., storeName=817ba5bcb98df62e5ed309caecfc1068/B, priority=13, startTime=1732710353977; duration=0sec 2024-11-27T12:25:54,458 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T12:25:54,458 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 817ba5bcb98df62e5ed309caecfc1068:B 2024-11-27T12:25:54,458 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T12:25:54,459 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36829 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T12:25:54,459 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HStore(1541): 817ba5bcb98df62e5ed309caecfc1068/C is initiating minor compaction (all files) 2024-11-27T12:25:54,459 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 817ba5bcb98df62e5ed309caecfc1068/C in TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068. 2024-11-27T12:25:54,459 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/C/0cef1e2389394b66a588690dd0d4ce30, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/C/4cd99bd20d0244168a58fd5af97647f0, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/C/5682906c1d364e31871391035db45e55] into tmpdir=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/.tmp, totalSize=36.0 K 2024-11-27T12:25:54,460 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.Compactor(225): Compacting 0cef1e2389394b66a588690dd0d4ce30, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1732710350673 2024-11-27T12:25:54,464 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.Compactor(225): Compacting 4cd99bd20d0244168a58fd5af97647f0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=192, earliestPutTs=1732710351915 2024-11-27T12:25:54,464 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/.tmp/A/c27c9d3696ea427a9422cb55df4e1c90 as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/A/c27c9d3696ea427a9422cb55df4e1c90 2024-11-27T12:25:54,464 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.Compactor(225): Compacting 5682906c1d364e31871391035db45e55, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1732710353115 2024-11-27T12:25:54,471 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 817ba5bcb98df62e5ed309caecfc1068/A of 817ba5bcb98df62e5ed309caecfc1068 into c27c9d3696ea427a9422cb55df4e1c90(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T12:25:54,471 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 817ba5bcb98df62e5ed309caecfc1068: 2024-11-27T12:25:54,471 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., storeName=817ba5bcb98df62e5ed309caecfc1068/A, priority=13, startTime=1732710353976; duration=0sec 2024-11-27T12:25:54,471 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T12:25:54,471 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 817ba5bcb98df62e5ed309caecfc1068:A 2024-11-27T12:25:54,487 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 817ba5bcb98df62e5ed309caecfc1068#C#compaction#326 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T12:25:54,487 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/.tmp/C/22dba8cf921e465cbf71326bfc2688bd is 50, key is test_row_0/C:col10/1732710353115/Put/seqid=0 2024-11-27T12:25:54,509 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742217_1393 (size=12629) 2024-11-27T12:25:54,514 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/.tmp/C/22dba8cf921e465cbf71326bfc2688bd as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/C/22dba8cf921e465cbf71326bfc2688bd 2024-11-27T12:25:54,522 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 817ba5bcb98df62e5ed309caecfc1068/C of 817ba5bcb98df62e5ed309caecfc1068 into 22dba8cf921e465cbf71326bfc2688bd(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T12:25:54,522 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 817ba5bcb98df62e5ed309caecfc1068: 2024-11-27T12:25:54,522 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., storeName=817ba5bcb98df62e5ed309caecfc1068/C, priority=13, startTime=1732710353977; duration=0sec 2024-11-27T12:25:54,522 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T12:25:54,522 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 817ba5bcb98df62e5ed309caecfc1068:C 2024-11-27T12:25:54,536 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=231, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/.tmp/A/04342a17b9bc486a965170f4df8af49a 2024-11-27T12:25:54,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/.tmp/B/4a530bb4637e43718be39a4180da8f07 is 50, key is test_row_0/B:col10/1732710353587/Put/seqid=0 2024-11-27T12:25:54,569 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742218_1394 (size=12151) 2024-11-27T12:25:54,572 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=231 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/.tmp/B/4a530bb4637e43718be39a4180da8f07 2024-11-27T12:25:54,581 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/.tmp/C/d51137eda6464409bb29c8c8027fd9ba is 50, key is test_row_0/C:col10/1732710353587/Put/seqid=0 2024-11-27T12:25:54,590 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742219_1395 (size=12151) 2024-11-27T12:25:54,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-27T12:25:54,642 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:54,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60076 deadline: 1732710414638, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:54,643 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:54,643 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:54,643 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:54,643 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:25:54,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60158 deadline: 1732710414638, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:54,643 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:54,643 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:54,643 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:25:54,954 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:54,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60076 deadline: 1732710414948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:54,954 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:54,954 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:54,954 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:54,954 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:25:54,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60158 deadline: 1732710414948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:54,954 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:54,954 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:54,954 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:25:54,990 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=231 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/.tmp/C/d51137eda6464409bb29c8c8027fd9ba 2024-11-27T12:25:54,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/.tmp/A/04342a17b9bc486a965170f4df8af49a as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/A/04342a17b9bc486a965170f4df8af49a 2024-11-27T12:25:55,003 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/A/04342a17b9bc486a965170f4df8af49a, entries=150, sequenceid=231, filesize=30.4 K 2024-11-27T12:25:55,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/.tmp/B/4a530bb4637e43718be39a4180da8f07 as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/B/4a530bb4637e43718be39a4180da8f07 2024-11-27T12:25:55,008 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/B/4a530bb4637e43718be39a4180da8f07, entries=150, sequenceid=231, filesize=11.9 K 2024-11-27T12:25:55,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/.tmp/C/d51137eda6464409bb29c8c8027fd9ba as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/C/d51137eda6464409bb29c8c8027fd9ba 2024-11-27T12:25:55,014 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/C/d51137eda6464409bb29c8c8027fd9ba, entries=150, sequenceid=231, filesize=11.9 K 2024-11-27T12:25:55,015 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=93.93 KB/96180 for 817ba5bcb98df62e5ed309caecfc1068 in 943ms, sequenceid=231, compaction requested=false 2024-11-27T12:25:55,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2603): Flush status journal for 817ba5bcb98df62e5ed309caecfc1068: 2024-11-27T12:25:55,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068. 2024-11-27T12:25:55,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=110 2024-11-27T12:25:55,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.HMaster(4169): Remote procedure done, pid=110 2024-11-27T12:25:55,019 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=110, resume processing ppid=109 2024-11-27T12:25:55,019 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=110, ppid=109, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5560 sec 2024-11-27T12:25:55,020 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=109, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees in 1.5600 sec 2024-11-27T12:25:55,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(8855): Flush requested on 817ba5bcb98df62e5ed309caecfc1068 2024-11-27T12:25:55,464 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T12:25:55,464 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 817ba5bcb98df62e5ed309caecfc1068 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-11-27T12:25:55,473 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127b0663af5fb11486e8d0019189b9f754d_817ba5bcb98df62e5ed309caecfc1068 is 50, key is test_row_0/A:col10/1732710354278/Put/seqid=0 2024-11-27T12:25:55,481 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742220_1396 (size=17284) 2024-11-27T12:25:55,562 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:55,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60158 deadline: 1732710415551, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:55,562 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:55,563 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:55,563 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:55,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60076 deadline: 1732710415552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:55,563 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:25:55,563 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:55,563 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:55,563 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:25:55,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-27T12:25:55,608 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T12:25:55,609 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-27T12:25:55,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] procedure2.ProcedureExecutor(1139): Stored pid=111, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees 2024-11-27T12:25:55,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-27T12:25:55,611 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=111, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T12:25:55,612 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=111, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T12:25:55,612 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=112, ppid=111, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T12:25:55,671 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:55,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60158 deadline: 1732710415668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:55,671 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:55,671 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:55,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60076 deadline: 1732710415668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:55,671 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:55,671 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:25:55,671 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:55,672 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:55,672 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:25:55,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-27T12:25:55,764 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-27T12:25:55,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068. 2024-11-27T12:25:55,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068. as already flushing 2024-11-27T12:25:55,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068. 2024-11-27T12:25:55,764 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 817ba5bcb98df62e5ed309caecfc1068, NAME => 'TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:25:55,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 817ba5bcb98df62e5ed309caecfc1068, NAME => 'TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:25:55,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 817ba5bcb98df62e5ed309caecfc1068, NAME => 'TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 817ba5bcb98df62e5ed309caecfc1068, NAME => 'TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:25:55,882 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:25:55,886 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127b0663af5fb11486e8d0019189b9f754d_817ba5bcb98df62e5ed309caecfc1068 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127b0663af5fb11486e8d0019189b9f754d_817ba5bcb98df62e5ed309caecfc1068 2024-11-27T12:25:55,886 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:55,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60158 deadline: 1732710415878, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:55,887 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:55,887 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:55,887 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:55,887 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:25:55,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60076 deadline: 1732710415878, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:55,887 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:55,887 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:55,887 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:25:55,888 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/.tmp/A/ace871bd4ece4f49adc5ed2c8c8b0acc, store: [table=TestAcidGuarantees family=A region=817ba5bcb98df62e5ed309caecfc1068] 2024-11-27T12:25:55,889 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/.tmp/A/ace871bd4ece4f49adc5ed2c8c8b0acc is 175, key is test_row_0/A:col10/1732710354278/Put/seqid=0 2024-11-27T12:25:55,892 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742221_1397 (size=48389) 2024-11-27T12:25:55,918 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-27T12:25:55,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068. 2024-11-27T12:25:55,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068. as already flushing 2024-11-27T12:25:55,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068. 2024-11-27T12:25:55,919 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 817ba5bcb98df62e5ed309caecfc1068, NAME => 'TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:25:55,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 817ba5bcb98df62e5ed309caecfc1068, NAME => 'TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:25:55,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 817ba5bcb98df62e5ed309caecfc1068, NAME => 'TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 817ba5bcb98df62e5ed309caecfc1068, NAME => 'TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:25:55,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-27T12:25:56,072 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-27T12:25:56,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068. 2024-11-27T12:25:56,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068. as already flushing 2024-11-27T12:25:56,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068. 2024-11-27T12:25:56,072 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 817ba5bcb98df62e5ed309caecfc1068, NAME => 'TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:25:56,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 817ba5bcb98df62e5ed309caecfc1068, NAME => 'TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:25:56,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 817ba5bcb98df62e5ed309caecfc1068, NAME => 'TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 817ba5bcb98df62e5ed309caecfc1068, NAME => 'TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:25:56,205 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:56,205 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:56,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60076 deadline: 1732710416197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:56,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60158 deadline: 1732710416197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:56,205 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:56,206 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:56,206 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:25:56,206 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:56,206 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:56,206 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:25:56,225 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-27T12:25:56,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068. 2024-11-27T12:25:56,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068. as already flushing 2024-11-27T12:25:56,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068. 2024-11-27T12:25:56,225 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 817ba5bcb98df62e5ed309caecfc1068, NAME => 'TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:25:56,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 817ba5bcb98df62e5ed309caecfc1068, NAME => 'TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:25:56,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 817ba5bcb98df62e5ed309caecfc1068, NAME => 'TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 817ba5bcb98df62e5ed309caecfc1068, NAME => 'TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:25:56,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-27T12:25:56,293 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=253, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/.tmp/A/ace871bd4ece4f49adc5ed2c8c8b0acc 2024-11-27T12:25:56,304 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/.tmp/B/e4b848ccea7845e880d062623b13a88b is 50, key is test_row_0/B:col10/1732710354278/Put/seqid=0 2024-11-27T12:25:56,311 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742222_1398 (size=12151) 2024-11-27T12:25:56,312 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=253 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/.tmp/B/e4b848ccea7845e880d062623b13a88b 2024-11-27T12:25:56,319 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/.tmp/C/c3b153f8d5f1428bb9042d1b43d7e63e is 50, key is test_row_0/C:col10/1732710354278/Put/seqid=0 2024-11-27T12:25:56,327 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742223_1399 (size=12151) 2024-11-27T12:25:56,378 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-27T12:25:56,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068. 2024-11-27T12:25:56,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068. as already flushing 2024-11-27T12:25:56,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068. 2024-11-27T12:25:56,379 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 817ba5bcb98df62e5ed309caecfc1068, NAME => 'TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:25:56,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 817ba5bcb98df62e5ed309caecfc1068, NAME => 'TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:25:56,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 817ba5bcb98df62e5ed309caecfc1068, NAME => 'TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 817ba5bcb98df62e5ed309caecfc1068, NAME => 'TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:25:56,531 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-27T12:25:56,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068. 2024-11-27T12:25:56,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068. as already flushing 2024-11-27T12:25:56,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068. 2024-11-27T12:25:56,531 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 817ba5bcb98df62e5ed309caecfc1068, NAME => 'TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:25:56,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 817ba5bcb98df62e5ed309caecfc1068, NAME => 'TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:25:56,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 817ba5bcb98df62e5ed309caecfc1068, NAME => 'TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 817ba5bcb98df62e5ed309caecfc1068, NAME => 'TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:25:56,684 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-27T12:25:56,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068. 2024-11-27T12:25:56,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068. as already flushing 2024-11-27T12:25:56,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068. 2024-11-27T12:25:56,685 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 817ba5bcb98df62e5ed309caecfc1068, NAME => 'TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:25:56,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 817ba5bcb98df62e5ed309caecfc1068, NAME => 'TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:25:56,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 817ba5bcb98df62e5ed309caecfc1068, NAME => 'TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 817ba5bcb98df62e5ed309caecfc1068, NAME => 'TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:25:56,727 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:56,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60158 deadline: 1732710416718, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:56,727 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:56,727 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:56,727 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:56,727 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:25:56,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60076 deadline: 1732710416718, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:56,728 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:56,728 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:56,728 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:25:56,729 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=253 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/.tmp/C/c3b153f8d5f1428bb9042d1b43d7e63e 2024-11-27T12:25:56,732 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/.tmp/A/ace871bd4ece4f49adc5ed2c8c8b0acc as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/A/ace871bd4ece4f49adc5ed2c8c8b0acc 2024-11-27T12:25:56,735 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/A/ace871bd4ece4f49adc5ed2c8c8b0acc, entries=250, sequenceid=253, filesize=47.3 K 2024-11-27T12:25:56,736 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/.tmp/B/e4b848ccea7845e880d062623b13a88b as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/B/e4b848ccea7845e880d062623b13a88b 2024-11-27T12:25:56,740 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/B/e4b848ccea7845e880d062623b13a88b, entries=150, sequenceid=253, filesize=11.9 K 2024-11-27T12:25:56,740 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/.tmp/C/c3b153f8d5f1428bb9042d1b43d7e63e as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/C/c3b153f8d5f1428bb9042d1b43d7e63e 2024-11-27T12:25:56,744 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/C/c3b153f8d5f1428bb9042d1b43d7e63e, entries=150, sequenceid=253, filesize=11.9 K 2024-11-27T12:25:56,745 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=93.93 KB/96180 for 817ba5bcb98df62e5ed309caecfc1068 in 1281ms, sequenceid=253, compaction requested=true 2024-11-27T12:25:56,745 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 817ba5bcb98df62e5ed309caecfc1068: 2024-11-27T12:25:56,745 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 817ba5bcb98df62e5ed309caecfc1068:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T12:25:56,745 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T12:25:56,745 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T12:25:56,745 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 817ba5bcb98df62e5ed309caecfc1068:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T12:25:56,745 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T12:25:56,745 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 817ba5bcb98df62e5ed309caecfc1068:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T12:25:56,745 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T12:25:56,745 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T12:25:56,746 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 111111 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T12:25:56,746 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HStore(1541): 817ba5bcb98df62e5ed309caecfc1068/A is initiating minor compaction (all files) 2024-11-27T12:25:56,746 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 817ba5bcb98df62e5ed309caecfc1068/A in TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068. 2024-11-27T12:25:56,746 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36965 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T12:25:56,746 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HStore(1541): 817ba5bcb98df62e5ed309caecfc1068/B is initiating minor compaction (all files) 2024-11-27T12:25:56,746 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 817ba5bcb98df62e5ed309caecfc1068/B in TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068. 2024-11-27T12:25:56,746 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/A/c27c9d3696ea427a9422cb55df4e1c90, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/A/04342a17b9bc486a965170f4df8af49a, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/A/ace871bd4ece4f49adc5ed2c8c8b0acc] into tmpdir=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/.tmp, totalSize=108.5 K 2024-11-27T12:25:56,746 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/B/6a5d768be57c4a91b822d6c4173755b9, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/B/4a530bb4637e43718be39a4180da8f07, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/B/e4b848ccea7845e880d062623b13a88b] into tmpdir=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/.tmp, totalSize=36.1 K 2024-11-27T12:25:56,746 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068. 2024-11-27T12:25:56,746 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068. files: [hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/A/c27c9d3696ea427a9422cb55df4e1c90, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/A/04342a17b9bc486a965170f4df8af49a, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/A/ace871bd4ece4f49adc5ed2c8c8b0acc] 2024-11-27T12:25:56,747 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.Compactor(225): Compacting 6a5d768be57c4a91b822d6c4173755b9, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1732710353115 2024-11-27T12:25:56,747 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] compactions.Compactor(225): Compacting c27c9d3696ea427a9422cb55df4e1c90, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1732710353115 2024-11-27T12:25:56,747 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.Compactor(225): Compacting 4a530bb4637e43718be39a4180da8f07, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=231, earliestPutTs=1732710353566 2024-11-27T12:25:56,747 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] compactions.Compactor(225): Compacting 04342a17b9bc486a965170f4df8af49a, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=231, earliestPutTs=1732710353566 2024-11-27T12:25:56,747 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.Compactor(225): Compacting e4b848ccea7845e880d062623b13a88b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1732710354278 2024-11-27T12:25:56,747 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] compactions.Compactor(225): Compacting ace871bd4ece4f49adc5ed2c8c8b0acc, keycount=250, bloomtype=ROW, size=47.3 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1732710354278 2024-11-27T12:25:56,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-27T12:25:56,754 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=817ba5bcb98df62e5ed309caecfc1068] 2024-11-27T12:25:56,766 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241127878a2febb0c4491f9eff0cd5e3219a64_817ba5bcb98df62e5ed309caecfc1068 store=[table=TestAcidGuarantees family=A region=817ba5bcb98df62e5ed309caecfc1068] 2024-11-27T12:25:56,767 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 817ba5bcb98df62e5ed309caecfc1068#B#compaction#333 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-27T12:25:56,767 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/.tmp/B/cdf013a848924db6985a5d7d82a3e7ed is 50, key is test_row_0/B:col10/1732710354278/Put/seqid=0 2024-11-27T12:25:56,768 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241127878a2febb0c4491f9eff0cd5e3219a64_817ba5bcb98df62e5ed309caecfc1068, store=[table=TestAcidGuarantees family=A region=817ba5bcb98df62e5ed309caecfc1068] 2024-11-27T12:25:56,768 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127878a2febb0c4491f9eff0cd5e3219a64_817ba5bcb98df62e5ed309caecfc1068 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=817ba5bcb98df62e5ed309caecfc1068] 2024-11-27T12:25:56,790 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742224_1400 (size=12765) 2024-11-27T12:25:56,796 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/.tmp/B/cdf013a848924db6985a5d7d82a3e7ed as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/B/cdf013a848924db6985a5d7d82a3e7ed 2024-11-27T12:25:56,800 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 817ba5bcb98df62e5ed309caecfc1068/B of 817ba5bcb98df62e5ed309caecfc1068 into cdf013a848924db6985a5d7d82a3e7ed(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T12:25:56,800 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 817ba5bcb98df62e5ed309caecfc1068: 2024-11-27T12:25:56,801 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., storeName=817ba5bcb98df62e5ed309caecfc1068/B, priority=13, startTime=1732710356745; duration=0sec 2024-11-27T12:25:56,801 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T12:25:56,801 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 817ba5bcb98df62e5ed309caecfc1068:B 2024-11-27T12:25:56,801 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T12:25:56,801 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742225_1401 (size=4469) 2024-11-27T12:25:56,802 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 817ba5bcb98df62e5ed309caecfc1068#A#compaction#332 average throughput is 0.51 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T12:25:56,802 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36931 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T12:25:56,802 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HStore(1541): 817ba5bcb98df62e5ed309caecfc1068/C is initiating minor compaction (all files) 2024-11-27T12:25:56,802 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 817ba5bcb98df62e5ed309caecfc1068/C in TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068. 2024-11-27T12:25:56,802 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/C/22dba8cf921e465cbf71326bfc2688bd, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/C/d51137eda6464409bb29c8c8027fd9ba, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/C/c3b153f8d5f1428bb9042d1b43d7e63e] into tmpdir=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/.tmp, totalSize=36.1 K 2024-11-27T12:25:56,803 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/.tmp/A/fe325587d17540b69d5a34f8c9672e8b is 175, key is test_row_0/A:col10/1732710354278/Put/seqid=0 2024-11-27T12:25:56,803 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.Compactor(225): Compacting 22dba8cf921e465cbf71326bfc2688bd, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1732710353115 2024-11-27T12:25:56,804 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.Compactor(225): Compacting d51137eda6464409bb29c8c8027fd9ba, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=231, earliestPutTs=1732710353566 2024-11-27T12:25:56,804 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.Compactor(225): Compacting c3b153f8d5f1428bb9042d1b43d7e63e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1732710354278 2024-11-27T12:25:56,817 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 817ba5bcb98df62e5ed309caecfc1068#C#compaction#334 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T12:25:56,817 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742226_1402 (size=31719) 2024-11-27T12:25:56,817 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/.tmp/C/17f4724423e2428fa7eca77a4d0e1d11 is 50, key is test_row_0/C:col10/1732710354278/Put/seqid=0 2024-11-27T12:25:56,827 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/.tmp/A/fe325587d17540b69d5a34f8c9672e8b as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/A/fe325587d17540b69d5a34f8c9672e8b 2024-11-27T12:25:56,837 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-27T12:25:56,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068. 2024-11-27T12:25:56,837 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2902): Flushing 817ba5bcb98df62e5ed309caecfc1068 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-11-27T12:25:56,839 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742227_1403 (size=12731) 2024-11-27T12:25:56,846 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 817ba5bcb98df62e5ed309caecfc1068/A of 817ba5bcb98df62e5ed309caecfc1068 into fe325587d17540b69d5a34f8c9672e8b(size=31.0 K), total size for store is 31.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T12:25:56,846 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 817ba5bcb98df62e5ed309caecfc1068: 2024-11-27T12:25:56,846 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., storeName=817ba5bcb98df62e5ed309caecfc1068/A, priority=13, startTime=1732710356745; duration=0sec 2024-11-27T12:25:56,846 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T12:25:56,846 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 817ba5bcb98df62e5ed309caecfc1068:A 2024-11-27T12:25:56,850 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/.tmp/C/17f4724423e2428fa7eca77a4d0e1d11 as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/C/17f4724423e2428fa7eca77a4d0e1d11 2024-11-27T12:25:56,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411275b22198619fd464bb94843bb6cc3668d_817ba5bcb98df62e5ed309caecfc1068 is 50, key is test_row_0/A:col10/1732710355548/Put/seqid=0 2024-11-27T12:25:56,871 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 817ba5bcb98df62e5ed309caecfc1068/C of 817ba5bcb98df62e5ed309caecfc1068 into 17f4724423e2428fa7eca77a4d0e1d11(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T12:25:56,871 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 817ba5bcb98df62e5ed309caecfc1068: 2024-11-27T12:25:56,871 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., storeName=817ba5bcb98df62e5ed309caecfc1068/C, priority=13, startTime=1732710356745; duration=0sec 2024-11-27T12:25:56,872 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T12:25:56,872 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 817ba5bcb98df62e5ed309caecfc1068:C 2024-11-27T12:25:56,876 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742228_1404 (size=12454) 2024-11-27T12:25:57,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:25:57,281 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411275b22198619fd464bb94843bb6cc3668d_817ba5bcb98df62e5ed309caecfc1068 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411275b22198619fd464bb94843bb6cc3668d_817ba5bcb98df62e5ed309caecfc1068 2024-11-27T12:25:57,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/.tmp/A/06799eb6ea9346df9566b325071ab910, store: [table=TestAcidGuarantees family=A region=817ba5bcb98df62e5ed309caecfc1068] 2024-11-27T12:25:57,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/.tmp/A/06799eb6ea9346df9566b325071ab910 is 175, key is test_row_0/A:col10/1732710355548/Put/seqid=0 2024-11-27T12:25:57,285 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742229_1405 (size=31255) 2024-11-27T12:25:57,686 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=271, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/.tmp/A/06799eb6ea9346df9566b325071ab910 2024-11-27T12:25:57,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/.tmp/B/8523250768df496a82a4fbbbf8bf75ce is 50, key is test_row_0/B:col10/1732710355548/Put/seqid=0 2024-11-27T12:25:57,709 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742230_1406 (size=12301) 2024-11-27T12:25:57,711 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=271 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/.tmp/B/8523250768df496a82a4fbbbf8bf75ce 2024-11-27T12:25:57,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/.tmp/C/26ac0c3e7c1f43f8bcfabf62a9aec6b4 is 50, key is test_row_0/C:col10/1732710355548/Put/seqid=0 2024-11-27T12:25:57,731 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742231_1407 (size=12301) 2024-11-27T12:25:57,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(8855): Flush requested on 817ba5bcb98df62e5ed309caecfc1068 2024-11-27T12:25:57,740 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068. as already flushing 2024-11-27T12:25:57,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-27T12:25:57,785 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:57,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60076 deadline: 1732710417783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:57,785 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:57,785 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:57,785 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:25:57,790 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:57,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60158 deadline: 1732710417784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:57,790 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:57,790 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:57,790 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:25:57,894 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:57,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60076 deadline: 1732710417891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:57,894 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:57,894 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:57,894 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:25:57,902 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:57,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60158 deadline: 1732710417897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:57,902 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:57,902 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:57,902 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:25:58,102 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:58,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60076 deadline: 1732710418098, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:58,103 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:58,103 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:58,103 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:25:58,115 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:58,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60158 deadline: 1732710418108, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:58,116 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:58,116 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:58,116 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:25:58,132 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=271 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/.tmp/C/26ac0c3e7c1f43f8bcfabf62a9aec6b4 2024-11-27T12:25:58,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/.tmp/A/06799eb6ea9346df9566b325071ab910 as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/A/06799eb6ea9346df9566b325071ab910 2024-11-27T12:25:58,139 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/A/06799eb6ea9346df9566b325071ab910, entries=150, sequenceid=271, filesize=30.5 K 2024-11-27T12:25:58,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/.tmp/B/8523250768df496a82a4fbbbf8bf75ce as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/B/8523250768df496a82a4fbbbf8bf75ce 2024-11-27T12:25:58,146 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/B/8523250768df496a82a4fbbbf8bf75ce, entries=150, sequenceid=271, filesize=12.0 K 2024-11-27T12:25:58,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/.tmp/C/26ac0c3e7c1f43f8bcfabf62a9aec6b4 as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/C/26ac0c3e7c1f43f8bcfabf62a9aec6b4 2024-11-27T12:25:58,150 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/C/26ac0c3e7c1f43f8bcfabf62a9aec6b4, entries=150, sequenceid=271, filesize=12.0 K 2024-11-27T12:25:58,151 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=107.34 KB/109920 for 817ba5bcb98df62e5ed309caecfc1068 in 1314ms, sequenceid=271, compaction requested=false 2024-11-27T12:25:58,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2603): Flush status journal for 817ba5bcb98df62e5ed309caecfc1068: 2024-11-27T12:25:58,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068. 2024-11-27T12:25:58,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=112 2024-11-27T12:25:58,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.HMaster(4169): Remote procedure done, pid=112 2024-11-27T12:25:58,154 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=112, resume processing ppid=111 2024-11-27T12:25:58,155 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=112, ppid=111, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.5410 sec 2024-11-27T12:25:58,156 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=111, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees in 2.5460 sec 2024-11-27T12:25:58,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(8855): Flush requested on 817ba5bcb98df62e5ed309caecfc1068 2024-11-27T12:25:58,411 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T12:25:58,411 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 817ba5bcb98df62e5ed309caecfc1068 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-27T12:25:58,418 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127e333e2e8f01a403aa9ffd9baadbfb143_817ba5bcb98df62e5ed309caecfc1068 is 50, key is test_row_0/A:col10/1732710358410/Put/seqid=0 2024-11-27T12:25:58,421 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742232_1408 (size=14994) 2024-11-27T12:25:58,464 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:58,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60076 deadline: 1732710418456, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:58,465 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:58,465 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:58,465 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:25:58,465 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:58,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60158 deadline: 1732710418460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:58,465 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:58,465 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:58,465 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:25:58,571 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:58,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60076 deadline: 1732710418568, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:58,572 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:58,572 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:58,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60158 deadline: 1732710418568, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:58,572 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:58,572 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:25:58,572 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:58,572 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:58,572 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:25:58,783 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:58,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60158 deadline: 1732710418778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:58,784 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:58,784 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:58,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60076 deadline: 1732710418778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:58,784 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:58,784 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:25:58,784 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:58,784 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:58,784 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:25:58,822 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:25:58,825 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127e333e2e8f01a403aa9ffd9baadbfb143_817ba5bcb98df62e5ed309caecfc1068 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127e333e2e8f01a403aa9ffd9baadbfb143_817ba5bcb98df62e5ed309caecfc1068 2024-11-27T12:25:58,826 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/.tmp/A/4fd838c0a2f04de4a588076cf79045e4, store: [table=TestAcidGuarantees family=A region=817ba5bcb98df62e5ed309caecfc1068] 2024-11-27T12:25:58,826 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/.tmp/A/4fd838c0a2f04de4a588076cf79045e4 is 175, key is test_row_0/A:col10/1732710358410/Put/seqid=0 2024-11-27T12:25:58,830 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742233_1409 (size=39949) 2024-11-27T12:25:58,831 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=293, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/.tmp/A/4fd838c0a2f04de4a588076cf79045e4 2024-11-27T12:25:58,837 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/.tmp/B/0a39785ddc7140c199a190e2f4118094 is 50, key is test_row_0/B:col10/1732710358410/Put/seqid=0 2024-11-27T12:25:58,841 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742234_1410 (size=12301) 2024-11-27T12:25:58,842 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=293 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/.tmp/B/0a39785ddc7140c199a190e2f4118094 2024-11-27T12:25:58,851 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/.tmp/C/10837574049744d2989f3bdcb9b94f2a is 50, key is test_row_0/C:col10/1732710358410/Put/seqid=0 2024-11-27T12:25:58,857 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742235_1411 (size=12301) 2024-11-27T12:25:59,094 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:59,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60076 deadline: 1732710419088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:59,094 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:59,094 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:59,094 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:59,094 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:25:59,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60158 deadline: 1732710419089, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:59,095 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:59,095 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:59,095 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:25:59,259 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=293 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/.tmp/C/10837574049744d2989f3bdcb9b94f2a 2024-11-27T12:25:59,262 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/.tmp/A/4fd838c0a2f04de4a588076cf79045e4 as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/A/4fd838c0a2f04de4a588076cf79045e4 2024-11-27T12:25:59,265 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/A/4fd838c0a2f04de4a588076cf79045e4, entries=200, sequenceid=293, filesize=39.0 K 2024-11-27T12:25:59,267 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/.tmp/B/0a39785ddc7140c199a190e2f4118094 as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/B/0a39785ddc7140c199a190e2f4118094 2024-11-27T12:25:59,271 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/B/0a39785ddc7140c199a190e2f4118094, entries=150, sequenceid=293, filesize=12.0 K 2024-11-27T12:25:59,272 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/.tmp/C/10837574049744d2989f3bdcb9b94f2a as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/C/10837574049744d2989f3bdcb9b94f2a 2024-11-27T12:25:59,279 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/C/10837574049744d2989f3bdcb9b94f2a, entries=150, sequenceid=293, filesize=12.0 K 2024-11-27T12:25:59,279 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=87.22 KB/89310 for 817ba5bcb98df62e5ed309caecfc1068 in 868ms, sequenceid=293, compaction requested=true 2024-11-27T12:25:59,280 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 817ba5bcb98df62e5ed309caecfc1068: 2024-11-27T12:25:59,280 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T12:25:59,280 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 817ba5bcb98df62e5ed309caecfc1068:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T12:25:59,280 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T12:25:59,280 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T12:25:59,280 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 817ba5bcb98df62e5ed309caecfc1068:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T12:25:59,280 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T12:25:59,280 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 817ba5bcb98df62e5ed309caecfc1068:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T12:25:59,280 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T12:25:59,281 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102923 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T12:25:59,281 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37367 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T12:25:59,281 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HStore(1541): 817ba5bcb98df62e5ed309caecfc1068/A is initiating minor compaction (all files) 2024-11-27T12:25:59,281 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HStore(1541): 817ba5bcb98df62e5ed309caecfc1068/B is initiating minor compaction (all files) 2024-11-27T12:25:59,281 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 817ba5bcb98df62e5ed309caecfc1068/A in TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068. 2024-11-27T12:25:59,281 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 817ba5bcb98df62e5ed309caecfc1068/B in TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068. 2024-11-27T12:25:59,281 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/A/fe325587d17540b69d5a34f8c9672e8b, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/A/06799eb6ea9346df9566b325071ab910, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/A/4fd838c0a2f04de4a588076cf79045e4] into tmpdir=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/.tmp, totalSize=100.5 K 2024-11-27T12:25:59,281 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/B/cdf013a848924db6985a5d7d82a3e7ed, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/B/8523250768df496a82a4fbbbf8bf75ce, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/B/0a39785ddc7140c199a190e2f4118094] into tmpdir=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/.tmp, totalSize=36.5 K 2024-11-27T12:25:59,281 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068. 2024-11-27T12:25:59,281 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068. files: [hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/A/fe325587d17540b69d5a34f8c9672e8b, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/A/06799eb6ea9346df9566b325071ab910, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/A/4fd838c0a2f04de4a588076cf79045e4] 2024-11-27T12:25:59,282 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] compactions.Compactor(225): Compacting fe325587d17540b69d5a34f8c9672e8b, keycount=150, bloomtype=ROW, size=31.0 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1732710354278 2024-11-27T12:25:59,282 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.Compactor(225): Compacting cdf013a848924db6985a5d7d82a3e7ed, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1732710354278 2024-11-27T12:25:59,282 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] compactions.Compactor(225): Compacting 06799eb6ea9346df9566b325071ab910, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=271, earliestPutTs=1732710355485 2024-11-27T12:25:59,282 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.Compactor(225): Compacting 8523250768df496a82a4fbbbf8bf75ce, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=271, earliestPutTs=1732710355485 2024-11-27T12:25:59,282 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4fd838c0a2f04de4a588076cf79045e4, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=293, earliestPutTs=1732710357739 2024-11-27T12:25:59,282 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.Compactor(225): Compacting 0a39785ddc7140c199a190e2f4118094, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=293, earliestPutTs=1732710357739 2024-11-27T12:25:59,286 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=817ba5bcb98df62e5ed309caecfc1068] 2024-11-27T12:25:59,289 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024112720ffee4e6bb1437f90edfcaf9b65a7b1_817ba5bcb98df62e5ed309caecfc1068 store=[table=TestAcidGuarantees family=A region=817ba5bcb98df62e5ed309caecfc1068] 2024-11-27T12:25:59,290 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 817ba5bcb98df62e5ed309caecfc1068#B#compaction#342 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-27T12:25:59,291 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/.tmp/B/d326c753aa0b4516a31dae3ed111329b is 50, key is test_row_0/B:col10/1732710358410/Put/seqid=0 2024-11-27T12:25:59,292 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024112720ffee4e6bb1437f90edfcaf9b65a7b1_817ba5bcb98df62e5ed309caecfc1068, store=[table=TestAcidGuarantees family=A region=817ba5bcb98df62e5ed309caecfc1068] 2024-11-27T12:25:59,292 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112720ffee4e6bb1437f90edfcaf9b65a7b1_817ba5bcb98df62e5ed309caecfc1068 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=817ba5bcb98df62e5ed309caecfc1068] 2024-11-27T12:25:59,311 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742236_1412 (size=13017) 2024-11-27T12:25:59,315 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742237_1413 (size=4469) 2024-11-27T12:25:59,317 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 817ba5bcb98df62e5ed309caecfc1068#A#compaction#341 average throughput is 0.79 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T12:25:59,317 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/.tmp/A/2f31f74e65114a999211e4044e73716d is 175, key is test_row_0/A:col10/1732710358410/Put/seqid=0 2024-11-27T12:25:59,321 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/.tmp/B/d326c753aa0b4516a31dae3ed111329b as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/B/d326c753aa0b4516a31dae3ed111329b 2024-11-27T12:25:59,325 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 817ba5bcb98df62e5ed309caecfc1068/B of 817ba5bcb98df62e5ed309caecfc1068 into d326c753aa0b4516a31dae3ed111329b(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T12:25:59,326 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 817ba5bcb98df62e5ed309caecfc1068: 2024-11-27T12:25:59,326 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., storeName=817ba5bcb98df62e5ed309caecfc1068/B, priority=13, startTime=1732710359280; duration=0sec 2024-11-27T12:25:59,326 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T12:25:59,326 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 817ba5bcb98df62e5ed309caecfc1068:B 2024-11-27T12:25:59,326 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T12:25:59,326 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37333 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T12:25:59,327 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HStore(1541): 817ba5bcb98df62e5ed309caecfc1068/C is initiating minor compaction (all files) 2024-11-27T12:25:59,327 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 817ba5bcb98df62e5ed309caecfc1068/C in TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068. 2024-11-27T12:25:59,327 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/C/17f4724423e2428fa7eca77a4d0e1d11, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/C/26ac0c3e7c1f43f8bcfabf62a9aec6b4, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/C/10837574049744d2989f3bdcb9b94f2a] into tmpdir=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/.tmp, totalSize=36.5 K 2024-11-27T12:25:59,327 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.Compactor(225): Compacting 17f4724423e2428fa7eca77a4d0e1d11, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1732710354278 2024-11-27T12:25:59,327 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.Compactor(225): Compacting 26ac0c3e7c1f43f8bcfabf62a9aec6b4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=271, earliestPutTs=1732710355485 2024-11-27T12:25:59,328 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.Compactor(225): Compacting 10837574049744d2989f3bdcb9b94f2a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=293, earliestPutTs=1732710357739 2024-11-27T12:25:59,334 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742238_1414 (size=31971) 2024-11-27T12:25:59,338 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 817ba5bcb98df62e5ed309caecfc1068#C#compaction#343 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T12:25:59,339 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/.tmp/C/bf26aceda510431db63543ece0a7bfc9 is 50, key is test_row_0/C:col10/1732710358410/Put/seqid=0 2024-11-27T12:25:59,364 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742239_1415 (size=12983) 2024-11-27T12:25:59,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(8855): Flush requested on 817ba5bcb98df62e5ed309caecfc1068 2024-11-27T12:25:59,600 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T12:25:59,600 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 817ba5bcb98df62e5ed309caecfc1068 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-11-27T12:25:59,606 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411278ae504ae0aff43d1a8aae0239376e5c5_817ba5bcb98df62e5ed309caecfc1068 is 50, key is test_row_0/A:col10/1732710359599/Put/seqid=0 2024-11-27T12:25:59,613 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742240_1416 (size=14994) 2024-11-27T12:25:59,665 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:59,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 176 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60076 deadline: 1732710419659, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:59,665 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:59,665 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:59,665 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:25:59,668 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:59,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60158 deadline: 1732710419661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:59,668 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:59,668 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:59,668 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:25:59,739 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/.tmp/A/2f31f74e65114a999211e4044e73716d as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/A/2f31f74e65114a999211e4044e73716d 2024-11-27T12:25:59,742 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 817ba5bcb98df62e5ed309caecfc1068/A of 817ba5bcb98df62e5ed309caecfc1068 into 2f31f74e65114a999211e4044e73716d(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T12:25:59,742 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 817ba5bcb98df62e5ed309caecfc1068: 2024-11-27T12:25:59,742 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., storeName=817ba5bcb98df62e5ed309caecfc1068/A, priority=13, startTime=1732710359280; duration=0sec 2024-11-27T12:25:59,743 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T12:25:59,743 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 817ba5bcb98df62e5ed309caecfc1068:A 2024-11-27T12:25:59,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-27T12:25:59,768 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T12:25:59,769 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-27T12:25:59,769 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/.tmp/C/bf26aceda510431db63543ece0a7bfc9 as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/C/bf26aceda510431db63543ece0a7bfc9 2024-11-27T12:25:59,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] procedure2.ProcedureExecutor(1139): Stored pid=113, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees 2024-11-27T12:25:59,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-27T12:25:59,770 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=113, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T12:25:59,771 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=113, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T12:25:59,771 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=114, ppid=113, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T12:25:59,773 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:59,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 177 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60076 deadline: 1732710419768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:59,773 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:59,773 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:59,773 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:25:59,782 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 817ba5bcb98df62e5ed309caecfc1068/C of 817ba5bcb98df62e5ed309caecfc1068 into bf26aceda510431db63543ece0a7bfc9(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T12:25:59,782 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 817ba5bcb98df62e5ed309caecfc1068: 2024-11-27T12:25:59,782 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., storeName=817ba5bcb98df62e5ed309caecfc1068/C, priority=13, startTime=1732710359280; duration=0sec 2024-11-27T12:25:59,782 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T12:25:59,782 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 817ba5bcb98df62e5ed309caecfc1068:C 2024-11-27T12:25:59,786 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:59,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60158 deadline: 1732710419778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:59,786 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:59,786 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:59,786 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:25:59,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-27T12:25:59,923 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-27T12:25:59,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068. 2024-11-27T12:25:59,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068. as already flushing 2024-11-27T12:25:59,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068. 2024-11-27T12:25:59,924 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 817ba5bcb98df62e5ed309caecfc1068, NAME => 'TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:25:59,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 817ba5bcb98df62e5ed309caecfc1068, NAME => 'TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:25:59,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 817ba5bcb98df62e5ed309caecfc1068, NAME => 'TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 817ba5bcb98df62e5ed309caecfc1068, NAME => 'TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:25:59,978 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:25:59,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 178 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60076 deadline: 1732710419978, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:25:59,979 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:59,979 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:25:59,979 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:26:00,004 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:00,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60158 deadline: 1732710419998, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:00,004 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:00,004 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:00,004 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:26:00,013 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:26:00,014 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-27T12:26:00,017 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411278ae504ae0aff43d1a8aae0239376e5c5_817ba5bcb98df62e5ed309caecfc1068 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411278ae504ae0aff43d1a8aae0239376e5c5_817ba5bcb98df62e5ed309caecfc1068 2024-11-27T12:26:00,017 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/.tmp/A/db9f7012f54a4303808bd43b9db6ee6d, store: [table=TestAcidGuarantees family=A region=817ba5bcb98df62e5ed309caecfc1068] 2024-11-27T12:26:00,018 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/.tmp/A/db9f7012f54a4303808bd43b9db6ee6d is 175, key is test_row_0/A:col10/1732710359599/Put/seqid=0 2024-11-27T12:26:00,023 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742241_1417 (size=39949) 2024-11-27T12:26:00,024 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=311, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/.tmp/A/db9f7012f54a4303808bd43b9db6ee6d 2024-11-27T12:26:00,032 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/.tmp/B/ea98b8357f7943a0ae0441fa0a5f3ace is 50, key is test_row_0/B:col10/1732710359599/Put/seqid=0 2024-11-27T12:26:00,041 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742242_1418 (size=12301) 2024-11-27T12:26:00,076 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-27T12:26:00,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068. 2024-11-27T12:26:00,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068. as already flushing 2024-11-27T12:26:00,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068. 2024-11-27T12:26:00,076 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 817ba5bcb98df62e5ed309caecfc1068, NAME => 'TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:26:00,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 817ba5bcb98df62e5ed309caecfc1068, NAME => 'TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:26:00,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 817ba5bcb98df62e5ed309caecfc1068, NAME => 'TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 817ba5bcb98df62e5ed309caecfc1068, NAME => 'TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:26:00,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-27T12:26:00,228 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-27T12:26:00,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068. 2024-11-27T12:26:00,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068. as already flushing 2024-11-27T12:26:00,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068. 2024-11-27T12:26:00,229 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 817ba5bcb98df62e5ed309caecfc1068, NAME => 'TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:26:00,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 817ba5bcb98df62e5ed309caecfc1068, NAME => 'TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:26:00,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 817ba5bcb98df62e5ed309caecfc1068, NAME => 'TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 817ba5bcb98df62e5ed309caecfc1068, NAME => 'TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:26:00,292 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:00,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60076 deadline: 1732710420288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:00,292 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:00,292 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:00,292 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:26:00,310 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:00,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60158 deadline: 1732710420308, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:00,311 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:00,311 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:00,311 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:26:00,381 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-27T12:26:00,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068. 2024-11-27T12:26:00,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068. as already flushing 2024-11-27T12:26:00,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068. 2024-11-27T12:26:00,381 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 817ba5bcb98df62e5ed309caecfc1068, NAME => 'TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:26:00,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 817ba5bcb98df62e5ed309caecfc1068, NAME => 'TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:26:00,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 817ba5bcb98df62e5ed309caecfc1068, NAME => 'TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 817ba5bcb98df62e5ed309caecfc1068, NAME => 'TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:26:00,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-27T12:26:00,442 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=311 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/.tmp/B/ea98b8357f7943a0ae0441fa0a5f3ace 2024-11-27T12:26:00,448 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/.tmp/C/653ebf67dca34f758a03e73ee82f2e28 is 50, key is test_row_0/C:col10/1732710359599/Put/seqid=0 2024-11-27T12:26:00,451 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742243_1419 (size=12301) 2024-11-27T12:26:00,451 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=311 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/.tmp/C/653ebf67dca34f758a03e73ee82f2e28 2024-11-27T12:26:00,455 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/.tmp/A/db9f7012f54a4303808bd43b9db6ee6d as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/A/db9f7012f54a4303808bd43b9db6ee6d 2024-11-27T12:26:00,458 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/A/db9f7012f54a4303808bd43b9db6ee6d, entries=200, sequenceid=311, filesize=39.0 K 2024-11-27T12:26:00,458 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/.tmp/B/ea98b8357f7943a0ae0441fa0a5f3ace as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/B/ea98b8357f7943a0ae0441fa0a5f3ace 2024-11-27T12:26:00,461 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/B/ea98b8357f7943a0ae0441fa0a5f3ace, entries=150, sequenceid=311, filesize=12.0 K 2024-11-27T12:26:00,462 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/.tmp/C/653ebf67dca34f758a03e73ee82f2e28 as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/C/653ebf67dca34f758a03e73ee82f2e28 2024-11-27T12:26:00,464 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/C/653ebf67dca34f758a03e73ee82f2e28, entries=150, sequenceid=311, filesize=12.0 K 2024-11-27T12:26:00,465 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=107.34 KB/109920 for 817ba5bcb98df62e5ed309caecfc1068 in 865ms, sequenceid=311, compaction requested=false 2024-11-27T12:26:00,465 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 817ba5bcb98df62e5ed309caecfc1068: 2024-11-27T12:26:00,534 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-27T12:26:00,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068. 2024-11-27T12:26:00,534 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2902): Flushing 817ba5bcb98df62e5ed309caecfc1068 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-11-27T12:26:00,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411275fda1fa758bf498294368c4df1ceaf3f_817ba5bcb98df62e5ed309caecfc1068 is 50, key is test_row_0/A:col10/1732710359660/Put/seqid=0 2024-11-27T12:26:00,545 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742244_1420 (size=12454) 2024-11-27T12:26:00,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(8855): Flush requested on 817ba5bcb98df62e5ed309caecfc1068 2024-11-27T12:26:00,802 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068. as already flushing 2024-11-27T12:26:00,854 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:00,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60158 deadline: 1732710420846, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:00,855 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:00,855 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:00,855 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:26:00,855 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:00,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 189 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60076 deadline: 1732710420851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:00,855 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:00,855 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:00,856 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:26:00,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-27T12:26:00,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:26:00,949 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411275fda1fa758bf498294368c4df1ceaf3f_817ba5bcb98df62e5ed309caecfc1068 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411275fda1fa758bf498294368c4df1ceaf3f_817ba5bcb98df62e5ed309caecfc1068 2024-11-27T12:26:00,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/.tmp/A/ebe05d131b5049db8bb71024445641f4, store: [table=TestAcidGuarantees family=A region=817ba5bcb98df62e5ed309caecfc1068] 2024-11-27T12:26:00,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/.tmp/A/ebe05d131b5049db8bb71024445641f4 is 175, key is test_row_0/A:col10/1732710359660/Put/seqid=0 2024-11-27T12:26:00,953 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742245_1421 (size=31255) 2024-11-27T12:26:00,959 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:00,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60158 deadline: 1732710420958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:00,960 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:00,960 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:00,960 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:00,960 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:26:00,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 190 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60076 deadline: 1732710420958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:00,960 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:00,960 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:00,960 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:26:01,175 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:01,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60158 deadline: 1732710421167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:01,176 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:01,176 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:01,176 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:26:01,176 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:01,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 191 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60076 deadline: 1732710421168, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:01,176 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:01,176 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:01,176 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:26:01,354 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=332, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/.tmp/A/ebe05d131b5049db8bb71024445641f4 2024-11-27T12:26:01,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/.tmp/B/c9018cd5ab984fe7b137d0fb4ce1e5c7 is 50, key is test_row_0/B:col10/1732710359660/Put/seqid=0 2024-11-27T12:26:01,364 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742246_1422 (size=12301) 2024-11-27T12:26:01,491 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:01,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 192 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60076 deadline: 1732710421488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:01,492 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:01,492 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:01,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60158 deadline: 1732710421488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:01,492 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:01,492 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:26:01,492 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:01,492 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:01,492 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:26:01,764 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=332 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/.tmp/B/c9018cd5ab984fe7b137d0fb4ce1e5c7 2024-11-27T12:26:01,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/.tmp/C/970d3b51cf434dcb9d02151914cac5d2 is 50, key is test_row_0/C:col10/1732710359660/Put/seqid=0 2024-11-27T12:26:01,777 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742247_1423 (size=12301) 2024-11-27T12:26:01,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-27T12:26:02,000 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:02,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 193 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60076 deadline: 1732710421997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:02,000 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:02,000 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:02,000 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:02,000 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:26:02,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60158 deadline: 1732710421998, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:02,001 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:02,001 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:02,001 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:26:02,177 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=332 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/.tmp/C/970d3b51cf434dcb9d02151914cac5d2 2024-11-27T12:26:02,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/.tmp/A/ebe05d131b5049db8bb71024445641f4 as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/A/ebe05d131b5049db8bb71024445641f4 2024-11-27T12:26:02,184 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/A/ebe05d131b5049db8bb71024445641f4, entries=150, sequenceid=332, filesize=30.5 K 2024-11-27T12:26:02,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/.tmp/B/c9018cd5ab984fe7b137d0fb4ce1e5c7 as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/B/c9018cd5ab984fe7b137d0fb4ce1e5c7 2024-11-27T12:26:02,187 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/B/c9018cd5ab984fe7b137d0fb4ce1e5c7, entries=150, sequenceid=332, filesize=12.0 K 2024-11-27T12:26:02,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/.tmp/C/970d3b51cf434dcb9d02151914cac5d2 as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/C/970d3b51cf434dcb9d02151914cac5d2 2024-11-27T12:26:02,191 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/C/970d3b51cf434dcb9d02151914cac5d2, entries=150, sequenceid=332, filesize=12.0 K 2024-11-27T12:26:02,192 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=93.93 KB/96180 for 817ba5bcb98df62e5ed309caecfc1068 in 1658ms, sequenceid=332, compaction requested=true 2024-11-27T12:26:02,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2603): Flush status journal for 817ba5bcb98df62e5ed309caecfc1068: 2024-11-27T12:26:02,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068. 2024-11-27T12:26:02,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=114 2024-11-27T12:26:02,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.HMaster(4169): Remote procedure done, pid=114 2024-11-27T12:26:02,194 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=114, resume processing ppid=113 2024-11-27T12:26:02,194 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=114, ppid=113, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.4220 sec 2024-11-27T12:26:02,196 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=113, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees in 2.4260 sec 2024-11-27T12:26:02,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(8855): Flush requested on 817ba5bcb98df62e5ed309caecfc1068 2024-11-27T12:26:02,724 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T12:26:02,724 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 817ba5bcb98df62e5ed309caecfc1068 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-11-27T12:26:02,729 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112759c5f5d5da8f4cb095cb37a80101f03d_817ba5bcb98df62e5ed309caecfc1068 is 50, key is test_row_0/A:col10/1732710362723/Put/seqid=0 2024-11-27T12:26:02,735 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742248_1424 (size=14994) 2024-11-27T12:26:02,782 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:02,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60108 deadline: 1732710422778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:02,782 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:02,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60134 deadline: 1732710422779, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:02,782 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:02,782 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:02,782 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:26:02,782 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 1ba5af43fd99,36251,1732710241631 for 'test_row_0' in 817ba5bcb98df62e5ed309caecfc1068 of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18336 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T12:26:02,783 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:02,783 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:02,783 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:26:02,789 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:02,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60150 deadline: 1732710422781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:02,789 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:02,789 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:02,789 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:26:02,896 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:02,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60134 deadline: 1732710422888, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:02,896 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:02,896 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:02,896 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:26:02,907 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:02,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60150 deadline: 1732710422898, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:02,907 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:02,907 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:02,907 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:26:03,015 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:03,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 194 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60076 deadline: 1732710423008, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:03,016 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 1ba5af43fd99,36251,1732710241631 for 'test_row_0' in 817ba5bcb98df62e5ed309caecfc1068 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2164 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T12:26:03,016 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:03,016 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:03,016 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:26:03,027 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:03,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60158 deadline: 1732710423018, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:03,027 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 1ba5af43fd99,36251,1732710241631 for 'test_row_2' in 817ba5bcb98df62e5ed309caecfc1068 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2180 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T12:26:03,027 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:03,027 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:03,027 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:26:03,116 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:03,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60134 deadline: 1732710423108, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:03,117 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:03,117 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:03,117 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:26:03,120 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:03,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60150 deadline: 1732710423118, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:03,120 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:03,120 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:03,120 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:26:03,136 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:26:03,139 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112759c5f5d5da8f4cb095cb37a80101f03d_817ba5bcb98df62e5ed309caecfc1068 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112759c5f5d5da8f4cb095cb37a80101f03d_817ba5bcb98df62e5ed309caecfc1068 2024-11-27T12:26:03,139 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/.tmp/A/e122c130b1fb4c4fa48f78225470de36, store: [table=TestAcidGuarantees family=A region=817ba5bcb98df62e5ed309caecfc1068] 2024-11-27T12:26:03,140 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/.tmp/A/e122c130b1fb4c4fa48f78225470de36 is 175, key is test_row_0/A:col10/1732710362723/Put/seqid=0 2024-11-27T12:26:03,142 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742249_1425 (size=39949) 2024-11-27T12:26:03,220 INFO [Thread-1519 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1519. 2024-11-27T12:26:03,221 DEBUG [Thread-1519 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T12:26:03,221 DEBUG [Thread-1519 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T12:26:03,221 DEBUG [Thread-1519 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T12:26:03,221 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T12:26:03,223 INFO [Thread-1513 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1513. 2024-11-27T12:26:03,223 DEBUG [Thread-1513 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T12:26:03,223 DEBUG [Thread-1513 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T12:26:03,223 DEBUG [Thread-1513 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T12:26:03,223 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T12:26:03,226 INFO [Thread-1521 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1521. 2024-11-27T12:26:03,226 DEBUG [Thread-1521 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T12:26:03,226 DEBUG [Thread-1521 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T12:26:03,226 DEBUG [Thread-1521 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T12:26:03,226 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T12:26:03,230 INFO [Thread-1517 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1517. 2024-11-27T12:26:03,230 DEBUG [Thread-1517 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T12:26:03,230 DEBUG [Thread-1517 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T12:26:03,230 DEBUG [Thread-1517 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T12:26:03,230 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T12:26:03,233 INFO [Thread-1515 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1515. 2024-11-27T12:26:03,233 DEBUG [Thread-1515 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T12:26:03,233 DEBUG [Thread-1515 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T12:26:03,233 DEBUG [Thread-1515 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T12:26:03,233 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T12:26:03,430 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:03,430 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:03,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60134 deadline: 1732710423429, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:03,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60150 deadline: 1732710423429, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:03,431 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:03,431 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:03,431 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:03,431 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:03,431 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:26:03,431 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:26:03,545 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=350, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/.tmp/A/e122c130b1fb4c4fa48f78225470de36 2024-11-27T12:26:03,559 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/.tmp/B/6f4e91191ac149f285429acf222a0f0c is 50, key is test_row_0/B:col10/1732710362723/Put/seqid=0 2024-11-27T12:26:03,563 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742250_1426 (size=12301) 2024-11-27T12:26:03,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-27T12:26:03,938 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T12:26:03,938 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:03,938 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:03,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60150 deadline: 1732710423938, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:03,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60134 deadline: 1732710423938, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:03,938 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:03,938 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:03,938 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:03,938 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:03,938 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:26:03,938 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:26:03,964 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=350 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/.tmp/B/6f4e91191ac149f285429acf222a0f0c 2024-11-27T12:26:03,973 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/.tmp/C/64b3a3761b064847b38d7030fbcf4f22 is 50, key is test_row_0/C:col10/1732710362723/Put/seqid=0 2024-11-27T12:26:03,977 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742251_1427 (size=12301) 2024-11-27T12:26:04,378 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=350 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/.tmp/C/64b3a3761b064847b38d7030fbcf4f22 2024-11-27T12:26:04,383 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/.tmp/A/e122c130b1fb4c4fa48f78225470de36 as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/A/e122c130b1fb4c4fa48f78225470de36 2024-11-27T12:26:04,388 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/A/e122c130b1fb4c4fa48f78225470de36, entries=200, sequenceid=350, filesize=39.0 K 2024-11-27T12:26:04,390 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/.tmp/B/6f4e91191ac149f285429acf222a0f0c as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/B/6f4e91191ac149f285429acf222a0f0c 2024-11-27T12:26:04,395 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/B/6f4e91191ac149f285429acf222a0f0c, entries=150, sequenceid=350, filesize=12.0 K 2024-11-27T12:26:04,396 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/.tmp/C/64b3a3761b064847b38d7030fbcf4f22 as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/C/64b3a3761b064847b38d7030fbcf4f22 2024-11-27T12:26:04,400 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/C/64b3a3761b064847b38d7030fbcf4f22, entries=150, sequenceid=350, filesize=12.0 K 2024-11-27T12:26:04,401 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=100.63 KB/103050 for 817ba5bcb98df62e5ed309caecfc1068 in 1677ms, sequenceid=350, compaction requested=true 2024-11-27T12:26:04,401 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 817ba5bcb98df62e5ed309caecfc1068: 2024-11-27T12:26:04,401 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 817ba5bcb98df62e5ed309caecfc1068:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T12:26:04,401 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T12:26:04,401 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-27T12:26:04,401 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 817ba5bcb98df62e5ed309caecfc1068:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T12:26:04,401 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T12:26:04,401 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 817ba5bcb98df62e5ed309caecfc1068:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T12:26:04,401 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-27T12:26:04,401 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T12:26:04,402 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49920 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-27T12:26:04,402 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 143124 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-27T12:26:04,402 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HStore(1541): 817ba5bcb98df62e5ed309caecfc1068/A is initiating minor compaction (all files) 2024-11-27T12:26:04,402 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HStore(1541): 817ba5bcb98df62e5ed309caecfc1068/B is initiating minor compaction (all files) 2024-11-27T12:26:04,402 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 817ba5bcb98df62e5ed309caecfc1068/B in TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068. 2024-11-27T12:26:04,402 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 817ba5bcb98df62e5ed309caecfc1068/A in TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068. 2024-11-27T12:26:04,402 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/B/d326c753aa0b4516a31dae3ed111329b, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/B/ea98b8357f7943a0ae0441fa0a5f3ace, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/B/c9018cd5ab984fe7b137d0fb4ce1e5c7, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/B/6f4e91191ac149f285429acf222a0f0c] into tmpdir=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/.tmp, totalSize=48.8 K 2024-11-27T12:26:04,402 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/A/2f31f74e65114a999211e4044e73716d, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/A/db9f7012f54a4303808bd43b9db6ee6d, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/A/ebe05d131b5049db8bb71024445641f4, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/A/e122c130b1fb4c4fa48f78225470de36] into tmpdir=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/.tmp, totalSize=139.8 K 2024-11-27T12:26:04,402 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068. 2024-11-27T12:26:04,403 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068. files: [hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/A/2f31f74e65114a999211e4044e73716d, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/A/db9f7012f54a4303808bd43b9db6ee6d, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/A/ebe05d131b5049db8bb71024445641f4, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/A/e122c130b1fb4c4fa48f78225470de36] 2024-11-27T12:26:04,403 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.Compactor(225): Compacting d326c753aa0b4516a31dae3ed111329b, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=293, earliestPutTs=1732710357739 2024-11-27T12:26:04,403 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2f31f74e65114a999211e4044e73716d, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=293, earliestPutTs=1732710357739 2024-11-27T12:26:04,403 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.Compactor(225): Compacting ea98b8357f7943a0ae0441fa0a5f3ace, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=311, earliestPutTs=1732710358446 2024-11-27T12:26:04,403 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] compactions.Compactor(225): Compacting db9f7012f54a4303808bd43b9db6ee6d, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=311, earliestPutTs=1732710358446 2024-11-27T12:26:04,403 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.Compactor(225): Compacting c9018cd5ab984fe7b137d0fb4ce1e5c7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=332, earliestPutTs=1732710359638 2024-11-27T12:26:04,403 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] compactions.Compactor(225): Compacting ebe05d131b5049db8bb71024445641f4, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=332, earliestPutTs=1732710359638 2024-11-27T12:26:04,403 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.Compactor(225): Compacting 6f4e91191ac149f285429acf222a0f0c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=350, earliestPutTs=1732710360844 2024-11-27T12:26:04,403 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] compactions.Compactor(225): Compacting e122c130b1fb4c4fa48f78225470de36, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=350, earliestPutTs=1732710360826 2024-11-27T12:26:04,411 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 817ba5bcb98df62e5ed309caecfc1068#B#compaction#353 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T12:26:04,412 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/.tmp/B/0413aa05788d42859bad57ec2df125f1 is 50, key is test_row_0/B:col10/1732710362723/Put/seqid=0 2024-11-27T12:26:04,413 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=817ba5bcb98df62e5ed309caecfc1068] 2024-11-27T12:26:04,417 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241127fd7ac1c3bfbc4b708a8ddfb1aee32c00_817ba5bcb98df62e5ed309caecfc1068 store=[table=TestAcidGuarantees family=A region=817ba5bcb98df62e5ed309caecfc1068] 2024-11-27T12:26:04,422 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241127fd7ac1c3bfbc4b708a8ddfb1aee32c00_817ba5bcb98df62e5ed309caecfc1068, store=[table=TestAcidGuarantees family=A region=817ba5bcb98df62e5ed309caecfc1068] 2024-11-27T12:26:04,422 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742252_1428 (size=13153) 2024-11-27T12:26:04,422 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127fd7ac1c3bfbc4b708a8ddfb1aee32c00_817ba5bcb98df62e5ed309caecfc1068 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=817ba5bcb98df62e5ed309caecfc1068] 2024-11-27T12:26:04,424 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742253_1429 (size=4469) 2024-11-27T12:26:04,679 DEBUG [regionserver/1ba5af43fd99:0.Chore.1 {}] throttle.PressureAwareCompactionThroughputController(103): CompactionPressure is 0.07692307692307693, tune throughput to 53.85 MB/second 2024-11-27T12:26:04,681 WARN [regionserver/1ba5af43fd99:0.Chore.1 {}] hbase.ExecutorStatusChore(69): RS_COMPACTED_FILES_DISCHARGER's size info, queued: 6, running: 1 2024-11-27T12:26:04,682 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/1ba5af43fd99:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/A/258a02271b394e0f92b422e4ffbbefa2, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/A/994e597073934b64b2026363f982f497, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/A/607c0ee7a4174c4694c7f374f5e820fc, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/A/4827b378778f4effad509797bdf5d641, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/A/ea9002f10f3c486eb77eb08b1d87af15, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/A/baa90092ebbe4b69b418c678b4d78644, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/A/40e5f80a08ee4b9993dff964912a7a7e, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/A/8889c0115b8a49eabfe31a6e3fd0c03a, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/A/969e6cbf7ac643aebcd72d6b8127478e, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/A/520e75c14a9e4e08b88eecde24c10354, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/A/e03e11f7b8334c7ba8c976fa7a6db5cf, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/A/14094e0bbda94990808621bc8ac2143a, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/A/4b2c4ec6b8b942ccb7d88937d3892b7b, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/A/b010378453de41a59802977b340a1c84, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/A/512b281878f04be1bb55668a4602e36e, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/A/c27c9d3696ea427a9422cb55df4e1c90, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/A/04342a17b9bc486a965170f4df8af49a, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/A/ace871bd4ece4f49adc5ed2c8c8b0acc, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/A/fe325587d17540b69d5a34f8c9672e8b, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/A/06799eb6ea9346df9566b325071ab910, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/A/4fd838c0a2f04de4a588076cf79045e4] to archive 2024-11-27T12:26:04,683 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/1ba5af43fd99:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-27T12:26:04,685 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/1ba5af43fd99:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/A/258a02271b394e0f92b422e4ffbbefa2 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/A/258a02271b394e0f92b422e4ffbbefa2 2024-11-27T12:26:04,687 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/1ba5af43fd99:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/A/994e597073934b64b2026363f982f497 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/A/994e597073934b64b2026363f982f497 2024-11-27T12:26:04,688 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/1ba5af43fd99:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/A/607c0ee7a4174c4694c7f374f5e820fc to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/A/607c0ee7a4174c4694c7f374f5e820fc 2024-11-27T12:26:04,689 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/1ba5af43fd99:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/A/4827b378778f4effad509797bdf5d641 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/A/4827b378778f4effad509797bdf5d641 2024-11-27T12:26:04,691 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/1ba5af43fd99:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/A/ea9002f10f3c486eb77eb08b1d87af15 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/A/ea9002f10f3c486eb77eb08b1d87af15 2024-11-27T12:26:04,692 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/1ba5af43fd99:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/A/baa90092ebbe4b69b418c678b4d78644 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/A/baa90092ebbe4b69b418c678b4d78644 2024-11-27T12:26:04,693 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/1ba5af43fd99:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/A/40e5f80a08ee4b9993dff964912a7a7e to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/A/40e5f80a08ee4b9993dff964912a7a7e 2024-11-27T12:26:04,694 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/1ba5af43fd99:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/A/8889c0115b8a49eabfe31a6e3fd0c03a to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/A/8889c0115b8a49eabfe31a6e3fd0c03a 2024-11-27T12:26:04,695 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/1ba5af43fd99:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/A/969e6cbf7ac643aebcd72d6b8127478e to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/A/969e6cbf7ac643aebcd72d6b8127478e 2024-11-27T12:26:04,696 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/1ba5af43fd99:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/A/520e75c14a9e4e08b88eecde24c10354 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/A/520e75c14a9e4e08b88eecde24c10354 2024-11-27T12:26:04,696 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/1ba5af43fd99:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/A/e03e11f7b8334c7ba8c976fa7a6db5cf to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/A/e03e11f7b8334c7ba8c976fa7a6db5cf 2024-11-27T12:26:04,697 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/1ba5af43fd99:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/A/14094e0bbda94990808621bc8ac2143a to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/A/14094e0bbda94990808621bc8ac2143a 2024-11-27T12:26:04,698 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/1ba5af43fd99:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/A/4b2c4ec6b8b942ccb7d88937d3892b7b to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/A/4b2c4ec6b8b942ccb7d88937d3892b7b 2024-11-27T12:26:04,699 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/1ba5af43fd99:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/A/b010378453de41a59802977b340a1c84 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/A/b010378453de41a59802977b340a1c84 2024-11-27T12:26:04,700 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/1ba5af43fd99:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/A/512b281878f04be1bb55668a4602e36e to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/A/512b281878f04be1bb55668a4602e36e 2024-11-27T12:26:04,701 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/1ba5af43fd99:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/A/c27c9d3696ea427a9422cb55df4e1c90 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/A/c27c9d3696ea427a9422cb55df4e1c90 2024-11-27T12:26:04,702 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/1ba5af43fd99:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/A/04342a17b9bc486a965170f4df8af49a to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/A/04342a17b9bc486a965170f4df8af49a 2024-11-27T12:26:04,702 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/1ba5af43fd99:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/A/ace871bd4ece4f49adc5ed2c8c8b0acc to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/A/ace871bd4ece4f49adc5ed2c8c8b0acc 2024-11-27T12:26:04,703 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/1ba5af43fd99:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/A/fe325587d17540b69d5a34f8c9672e8b to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/A/fe325587d17540b69d5a34f8c9672e8b 2024-11-27T12:26:04,704 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/1ba5af43fd99:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/A/06799eb6ea9346df9566b325071ab910 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/A/06799eb6ea9346df9566b325071ab910 2024-11-27T12:26:04,704 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/1ba5af43fd99:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/A/4fd838c0a2f04de4a588076cf79045e4 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/A/4fd838c0a2f04de4a588076cf79045e4 2024-11-27T12:26:04,707 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/1ba5af43fd99:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/B/c90ecc5435224a2b85feb3270cd47937, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/B/8449fbd039ad4ba0b3b1060dbf6dc7f7, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/B/8135cb7ee15943d8b70edc4caa48092b, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/B/0190efb6011340f0b3305ffe36521cf3, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/B/53450ea59c9a43d2af06c6c97d0f3321, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/B/52ca6d4fd2624927a2898f9b8f1a1e31, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/B/80af0fb64dfb4d81b80a23ff6e58f885, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/B/d01946b594b642509cc52b66195c09ea, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/B/649e0c1d90e9450aa634acdfa8fe0223, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/B/7bbe02f709b74eb08cf8016be0748378, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/B/caf4d68a8a974141b6385cc56e74324b, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/B/4cdd3531e02243b682fde13b024a3e23, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/B/2059facbbd2e477b93bb7521e62bb7f1, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/B/1315c53304f34655859df25e0a66925e, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/B/6a5d768be57c4a91b822d6c4173755b9, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/B/4c9cae6bfb014bb0a23800e3f126bf36, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/B/4a530bb4637e43718be39a4180da8f07, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/B/cdf013a848924db6985a5d7d82a3e7ed, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/B/e4b848ccea7845e880d062623b13a88b, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/B/8523250768df496a82a4fbbbf8bf75ce, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/B/0a39785ddc7140c199a190e2f4118094] to archive 2024-11-27T12:26:04,708 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/1ba5af43fd99:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-27T12:26:04,709 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/1ba5af43fd99:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/B/c90ecc5435224a2b85feb3270cd47937 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/B/c90ecc5435224a2b85feb3270cd47937 2024-11-27T12:26:04,710 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/1ba5af43fd99:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/B/8449fbd039ad4ba0b3b1060dbf6dc7f7 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/B/8449fbd039ad4ba0b3b1060dbf6dc7f7 2024-11-27T12:26:04,711 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/1ba5af43fd99:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/B/8135cb7ee15943d8b70edc4caa48092b to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/B/8135cb7ee15943d8b70edc4caa48092b 2024-11-27T12:26:04,712 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/1ba5af43fd99:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/B/0190efb6011340f0b3305ffe36521cf3 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/B/0190efb6011340f0b3305ffe36521cf3 2024-11-27T12:26:04,713 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/1ba5af43fd99:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/B/53450ea59c9a43d2af06c6c97d0f3321 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/B/53450ea59c9a43d2af06c6c97d0f3321 2024-11-27T12:26:04,714 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/1ba5af43fd99:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/B/52ca6d4fd2624927a2898f9b8f1a1e31 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/B/52ca6d4fd2624927a2898f9b8f1a1e31 2024-11-27T12:26:04,715 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/1ba5af43fd99:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/B/80af0fb64dfb4d81b80a23ff6e58f885 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/B/80af0fb64dfb4d81b80a23ff6e58f885 2024-11-27T12:26:04,715 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/1ba5af43fd99:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/B/d01946b594b642509cc52b66195c09ea to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/B/d01946b594b642509cc52b66195c09ea 2024-11-27T12:26:04,717 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/1ba5af43fd99:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/B/649e0c1d90e9450aa634acdfa8fe0223 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/B/649e0c1d90e9450aa634acdfa8fe0223 2024-11-27T12:26:04,718 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/1ba5af43fd99:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/B/7bbe02f709b74eb08cf8016be0748378 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/B/7bbe02f709b74eb08cf8016be0748378 2024-11-27T12:26:04,719 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/1ba5af43fd99:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/B/caf4d68a8a974141b6385cc56e74324b to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/B/caf4d68a8a974141b6385cc56e74324b 2024-11-27T12:26:04,720 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/1ba5af43fd99:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/B/4cdd3531e02243b682fde13b024a3e23 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/B/4cdd3531e02243b682fde13b024a3e23 2024-11-27T12:26:04,721 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/1ba5af43fd99:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/B/2059facbbd2e477b93bb7521e62bb7f1 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/B/2059facbbd2e477b93bb7521e62bb7f1 2024-11-27T12:26:04,722 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/1ba5af43fd99:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/B/1315c53304f34655859df25e0a66925e to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/B/1315c53304f34655859df25e0a66925e 2024-11-27T12:26:04,724 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/1ba5af43fd99:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/B/6a5d768be57c4a91b822d6c4173755b9 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/B/6a5d768be57c4a91b822d6c4173755b9 2024-11-27T12:26:04,725 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/1ba5af43fd99:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/B/4c9cae6bfb014bb0a23800e3f126bf36 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/B/4c9cae6bfb014bb0a23800e3f126bf36 2024-11-27T12:26:04,726 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/1ba5af43fd99:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/B/4a530bb4637e43718be39a4180da8f07 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/B/4a530bb4637e43718be39a4180da8f07 2024-11-27T12:26:04,727 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/1ba5af43fd99:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/B/cdf013a848924db6985a5d7d82a3e7ed to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/B/cdf013a848924db6985a5d7d82a3e7ed 2024-11-27T12:26:04,729 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/1ba5af43fd99:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/B/e4b848ccea7845e880d062623b13a88b to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/B/e4b848ccea7845e880d062623b13a88b 2024-11-27T12:26:04,730 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/1ba5af43fd99:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/B/8523250768df496a82a4fbbbf8bf75ce to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/B/8523250768df496a82a4fbbbf8bf75ce 2024-11-27T12:26:04,731 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/1ba5af43fd99:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/B/0a39785ddc7140c199a190e2f4118094 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/B/0a39785ddc7140c199a190e2f4118094 2024-11-27T12:26:04,736 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/1ba5af43fd99:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/C/075f9746e10c486787b659208dfac1be, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/C/d7695678bc774d8382f9883e6c13afb2, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/C/c8fcd07d85034991a43cfca6db974b92, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/C/796725ff60204c8b9c1337c461d290ef, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/C/5cdd1c6857d2443ea2992b9a022d5c65, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/C/fcccf1ce41c947f3a4f6c6186ee2537f, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/C/06b2c9f79fd940c9965d97cf9b259ab4, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/C/92bb4ff9bdaf4e2a949f294c39774a8b, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/C/8e22b3175f5e4838bea3f7f13de4e688, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/C/ddfcb19b8e8a4e9f95f465cf8750535d, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/C/0cef1e2389394b66a588690dd0d4ce30, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/C/141608c0d11a41ba99a9142cd9b4f431, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/C/4cd99bd20d0244168a58fd5af97647f0, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/C/22dba8cf921e465cbf71326bfc2688bd, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/C/5682906c1d364e31871391035db45e55, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/C/d51137eda6464409bb29c8c8027fd9ba, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/C/17f4724423e2428fa7eca77a4d0e1d11, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/C/c3b153f8d5f1428bb9042d1b43d7e63e, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/C/26ac0c3e7c1f43f8bcfabf62a9aec6b4, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/C/10837574049744d2989f3bdcb9b94f2a] to archive 2024-11-27T12:26:04,737 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/1ba5af43fd99:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-27T12:26:04,739 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/1ba5af43fd99:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/C/075f9746e10c486787b659208dfac1be to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/C/075f9746e10c486787b659208dfac1be 2024-11-27T12:26:04,740 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/1ba5af43fd99:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/C/d7695678bc774d8382f9883e6c13afb2 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/C/d7695678bc774d8382f9883e6c13afb2 2024-11-27T12:26:04,742 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/1ba5af43fd99:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/C/c8fcd07d85034991a43cfca6db974b92 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/C/c8fcd07d85034991a43cfca6db974b92 2024-11-27T12:26:04,743 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/1ba5af43fd99:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/C/796725ff60204c8b9c1337c461d290ef to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/C/796725ff60204c8b9c1337c461d290ef 2024-11-27T12:26:04,745 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/1ba5af43fd99:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/C/5cdd1c6857d2443ea2992b9a022d5c65 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/C/5cdd1c6857d2443ea2992b9a022d5c65 2024-11-27T12:26:04,746 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/1ba5af43fd99:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/C/fcccf1ce41c947f3a4f6c6186ee2537f to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/C/fcccf1ce41c947f3a4f6c6186ee2537f 2024-11-27T12:26:04,748 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/1ba5af43fd99:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/C/06b2c9f79fd940c9965d97cf9b259ab4 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/C/06b2c9f79fd940c9965d97cf9b259ab4 2024-11-27T12:26:04,749 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/1ba5af43fd99:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/C/92bb4ff9bdaf4e2a949f294c39774a8b to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/C/92bb4ff9bdaf4e2a949f294c39774a8b 2024-11-27T12:26:04,751 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/1ba5af43fd99:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/C/8e22b3175f5e4838bea3f7f13de4e688 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/C/8e22b3175f5e4838bea3f7f13de4e688 2024-11-27T12:26:04,752 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/1ba5af43fd99:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/C/ddfcb19b8e8a4e9f95f465cf8750535d to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/C/ddfcb19b8e8a4e9f95f465cf8750535d 2024-11-27T12:26:04,754 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/1ba5af43fd99:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/C/0cef1e2389394b66a588690dd0d4ce30 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/C/0cef1e2389394b66a588690dd0d4ce30 2024-11-27T12:26:04,755 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/1ba5af43fd99:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/C/141608c0d11a41ba99a9142cd9b4f431 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/C/141608c0d11a41ba99a9142cd9b4f431 2024-11-27T12:26:04,756 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/1ba5af43fd99:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/C/4cd99bd20d0244168a58fd5af97647f0 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/C/4cd99bd20d0244168a58fd5af97647f0 2024-11-27T12:26:04,758 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/1ba5af43fd99:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/C/22dba8cf921e465cbf71326bfc2688bd to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/C/22dba8cf921e465cbf71326bfc2688bd 2024-11-27T12:26:04,759 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/1ba5af43fd99:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/C/5682906c1d364e31871391035db45e55 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/C/5682906c1d364e31871391035db45e55 2024-11-27T12:26:04,760 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/1ba5af43fd99:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/C/d51137eda6464409bb29c8c8027fd9ba to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/C/d51137eda6464409bb29c8c8027fd9ba 2024-11-27T12:26:04,762 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/1ba5af43fd99:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/C/17f4724423e2428fa7eca77a4d0e1d11 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/C/17f4724423e2428fa7eca77a4d0e1d11 2024-11-27T12:26:04,763 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/1ba5af43fd99:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/C/c3b153f8d5f1428bb9042d1b43d7e63e to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/C/c3b153f8d5f1428bb9042d1b43d7e63e 2024-11-27T12:26:04,764 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/1ba5af43fd99:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/C/26ac0c3e7c1f43f8bcfabf62a9aec6b4 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/C/26ac0c3e7c1f43f8bcfabf62a9aec6b4 2024-11-27T12:26:04,765 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/1ba5af43fd99:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/C/10837574049744d2989f3bdcb9b94f2a to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/C/10837574049744d2989f3bdcb9b94f2a 2024-11-27T12:26:04,827 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 817ba5bcb98df62e5ed309caecfc1068#A#compaction#354 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-27T12:26:04,828 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/.tmp/A/4cda6e8e585143739e4b4c0e51f69b5e is 175, key is test_row_0/A:col10/1732710362723/Put/seqid=0 2024-11-27T12:26:04,832 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/.tmp/B/0413aa05788d42859bad57ec2df125f1 as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/B/0413aa05788d42859bad57ec2df125f1 2024-11-27T12:26:04,833 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742254_1430 (size=31393) 2024-11-27T12:26:04,837 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 817ba5bcb98df62e5ed309caecfc1068/B of 817ba5bcb98df62e5ed309caecfc1068 into 0413aa05788d42859bad57ec2df125f1(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T12:26:04,837 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 817ba5bcb98df62e5ed309caecfc1068: 2024-11-27T12:26:04,838 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., storeName=817ba5bcb98df62e5ed309caecfc1068/B, priority=12, startTime=1732710364401; duration=0sec 2024-11-27T12:26:04,838 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T12:26:04,838 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 817ba5bcb98df62e5ed309caecfc1068:B 2024-11-27T12:26:04,838 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-27T12:26:04,839 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49886 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-27T12:26:04,839 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HStore(1541): 817ba5bcb98df62e5ed309caecfc1068/C is initiating minor compaction (all files) 2024-11-27T12:26:04,839 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 817ba5bcb98df62e5ed309caecfc1068/C in TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068. 2024-11-27T12:26:04,839 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/C/bf26aceda510431db63543ece0a7bfc9, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/C/653ebf67dca34f758a03e73ee82f2e28, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/C/970d3b51cf434dcb9d02151914cac5d2, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/C/64b3a3761b064847b38d7030fbcf4f22] into tmpdir=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/.tmp, totalSize=48.7 K 2024-11-27T12:26:04,840 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.Compactor(225): Compacting bf26aceda510431db63543ece0a7bfc9, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=293, earliestPutTs=1732710357739 2024-11-27T12:26:04,840 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.Compactor(225): Compacting 653ebf67dca34f758a03e73ee82f2e28, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=311, earliestPutTs=1732710358446 2024-11-27T12:26:04,841 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.Compactor(225): Compacting 970d3b51cf434dcb9d02151914cac5d2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=332, earliestPutTs=1732710359638 2024-11-27T12:26:04,841 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.Compactor(225): Compacting 64b3a3761b064847b38d7030fbcf4f22, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=350, earliestPutTs=1732710360844 2024-11-27T12:26:04,851 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 817ba5bcb98df62e5ed309caecfc1068#C#compaction#355 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-27T12:26:04,851 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/.tmp/C/a5ee312d754e4fe490f95dee90ed121c is 50, key is test_row_0/C:col10/1732710362723/Put/seqid=0 2024-11-27T12:26:04,855 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742255_1431 (size=12439) 2024-11-27T12:26:04,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(8855): Flush requested on 817ba5bcb98df62e5ed309caecfc1068 2024-11-27T12:26:04,951 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T12:26:04,951 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 817ba5bcb98df62e5ed309caecfc1068 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-27T12:26:04,952 INFO [Thread-1510 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1510. 2024-11-27T12:26:04,952 INFO [Thread-1508 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1508. 2024-11-27T12:26:04,952 DEBUG [Thread-1510 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T12:26:04,952 DEBUG [Thread-1510 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T12:26:04,952 DEBUG [Thread-1508 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T12:26:04,952 DEBUG [Thread-1508 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T12:26:04,952 DEBUG [Thread-1510 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T12:26:04,952 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T12:26:04,952 DEBUG [Thread-1508 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T12:26:04,952 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T12:26:04,961 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127604389cd26114820869c01a67652df5f_817ba5bcb98df62e5ed309caecfc1068 is 50, key is test_row_0/A:col10/1732710362767/Put/seqid=0 2024-11-27T12:26:04,964 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742256_1432 (size=12454) 2024-11-27T12:26:05,040 INFO [Thread-1502 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1502. 2024-11-27T12:26:05,040 DEBUG [Thread-1502 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T12:26:05,040 DEBUG [Thread-1502 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T12:26:05,040 DEBUG [Thread-1502 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T12:26:05,040 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T12:26:05,051 INFO [Thread-1504 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1504. 2024-11-27T12:26:05,051 DEBUG [Thread-1504 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T12:26:05,051 DEBUG [Thread-1504 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T12:26:05,051 DEBUG [Thread-1504 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T12:26:05,051 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T12:26:05,245 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/.tmp/A/4cda6e8e585143739e4b4c0e51f69b5e as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/A/4cda6e8e585143739e4b4c0e51f69b5e 2024-11-27T12:26:05,253 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 817ba5bcb98df62e5ed309caecfc1068/A of 817ba5bcb98df62e5ed309caecfc1068 into 4cda6e8e585143739e4b4c0e51f69b5e(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T12:26:05,254 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 817ba5bcb98df62e5ed309caecfc1068: 2024-11-27T12:26:05,254 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., storeName=817ba5bcb98df62e5ed309caecfc1068/A, priority=12, startTime=1732710364401; duration=0sec 2024-11-27T12:26:05,254 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T12:26:05,254 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 817ba5bcb98df62e5ed309caecfc1068:A 2024-11-27T12:26:05,261 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/.tmp/C/a5ee312d754e4fe490f95dee90ed121c as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/C/a5ee312d754e4fe490f95dee90ed121c 2024-11-27T12:26:05,267 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 817ba5bcb98df62e5ed309caecfc1068/C of 817ba5bcb98df62e5ed309caecfc1068 into a5ee312d754e4fe490f95dee90ed121c(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T12:26:05,267 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 817ba5bcb98df62e5ed309caecfc1068: 2024-11-27T12:26:05,267 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., storeName=817ba5bcb98df62e5ed309caecfc1068/C, priority=12, startTime=1732710364401; duration=0sec 2024-11-27T12:26:05,267 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T12:26:05,267 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 817ba5bcb98df62e5ed309caecfc1068:C 2024-11-27T12:26:05,365 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:26:05,372 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127604389cd26114820869c01a67652df5f_817ba5bcb98df62e5ed309caecfc1068 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127604389cd26114820869c01a67652df5f_817ba5bcb98df62e5ed309caecfc1068 2024-11-27T12:26:05,374 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/.tmp/A/19f4dc00ecc4454d98bde82fdc959a51, store: [table=TestAcidGuarantees family=A region=817ba5bcb98df62e5ed309caecfc1068] 2024-11-27T12:26:05,375 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/.tmp/A/19f4dc00ecc4454d98bde82fdc959a51 is 175, key is test_row_0/A:col10/1732710362767/Put/seqid=0 2024-11-27T12:26:05,379 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742257_1433 (size=31255) 2024-11-27T12:26:05,780 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=371, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/.tmp/A/19f4dc00ecc4454d98bde82fdc959a51 2024-11-27T12:26:05,786 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/.tmp/B/5fae9d0aef344578adfdb661f96e7ced is 50, key is test_row_0/B:col10/1732710362767/Put/seqid=0 2024-11-27T12:26:05,788 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742258_1434 (size=12301) 2024-11-27T12:26:06,189 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=371 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/.tmp/B/5fae9d0aef344578adfdb661f96e7ced 2024-11-27T12:26:06,195 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/.tmp/C/fa9e0e5b230143a2af0eab553aa6f756 is 50, key is test_row_0/C:col10/1732710362767/Put/seqid=0 2024-11-27T12:26:06,198 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742259_1435 (size=12301) 2024-11-27T12:26:06,600 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=371 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/.tmp/C/fa9e0e5b230143a2af0eab553aa6f756 2024-11-27T12:26:06,606 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/.tmp/A/19f4dc00ecc4454d98bde82fdc959a51 as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/A/19f4dc00ecc4454d98bde82fdc959a51 2024-11-27T12:26:06,611 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/A/19f4dc00ecc4454d98bde82fdc959a51, entries=150, sequenceid=371, filesize=30.5 K 2024-11-27T12:26:06,612 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/.tmp/B/5fae9d0aef344578adfdb661f96e7ced as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/B/5fae9d0aef344578adfdb661f96e7ced 2024-11-27T12:26:06,616 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/B/5fae9d0aef344578adfdb661f96e7ced, entries=150, sequenceid=371, filesize=12.0 K 2024-11-27T12:26:06,617 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/.tmp/C/fa9e0e5b230143a2af0eab553aa6f756 as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/C/fa9e0e5b230143a2af0eab553aa6f756 2024-11-27T12:26:06,621 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/C/fa9e0e5b230143a2af0eab553aa6f756, entries=150, sequenceid=371, filesize=12.0 K 2024-11-27T12:26:06,622 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=13.42 KB/13740 for 817ba5bcb98df62e5ed309caecfc1068 in 1671ms, sequenceid=371, compaction requested=false 2024-11-27T12:26:06,622 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 817ba5bcb98df62e5ed309caecfc1068: 2024-11-27T12:26:12,822 INFO [Thread-1506 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1506. 2024-11-27T12:26:12,822 DEBUG [Thread-1506 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T12:26:12,822 DEBUG [Thread-1506 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T12:26:12,823 DEBUG [Thread-1506 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T12:26:12,823 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T12:26:12,823 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-27T12:26:12,823 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 135 2024-11-27T12:26:12,824 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 116 2024-11-27T12:26:12,824 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 11 2024-11-27T12:26:12,824 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 15 2024-11-27T12:26:12,824 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 15 2024-11-27T12:26:12,824 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-27T12:26:12,824 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-27T12:26:12,824 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1317 2024-11-27T12:26:12,824 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3949 rows 2024-11-27T12:26:12,824 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1323 2024-11-27T12:26:12,824 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3961 rows 2024-11-27T12:26:12,824 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1319 2024-11-27T12:26:12,824 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3956 rows 2024-11-27T12:26:12,824 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1328 2024-11-27T12:26:12,824 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3982 rows 2024-11-27T12:26:12,824 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1321 2024-11-27T12:26:12,824 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3961 rows 2024-11-27T12:26:12,824 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-27T12:26:12,825 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobScanAtomicity(AcidGuaranteesTestBase.java:127) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-27T12:26:12,825 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T12:26:12,825 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T12:26:12,825 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T12:26:12,827 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-11-27T12:26:12,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] procedure2.ProcedureExecutor(1139): Stored pid=115, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-27T12:26:12,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-27T12:26:12,831 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732710372831"}]},"ts":"1732710372831"} 2024-11-27T12:26:12,832 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-27T12:26:12,832 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-27T12:26:12,833 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=116, ppid=115, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-27T12:26:12,835 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=117, ppid=116, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=817ba5bcb98df62e5ed309caecfc1068, UNASSIGN}] 2024-11-27T12:26:12,836 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=117, ppid=116, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=817ba5bcb98df62e5ed309caecfc1068, UNASSIGN 2024-11-27T12:26:12,836 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=117 updating hbase:meta row=817ba5bcb98df62e5ed309caecfc1068, regionState=CLOSING, regionLocation=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:12,838 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=117, ppid=116, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=817ba5bcb98df62e5ed309caecfc1068, UNASSIGN because future has completed 2024-11-27T12:26:12,838 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-27T12:26:12,838 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=118, ppid=117, state=RUNNABLE, hasLock=false; CloseRegionProcedure 817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631}] 2024-11-27T12:26:12,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-27T12:26:12,991 INFO [RS_CLOSE_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] handler.UnassignRegionHandler(122): Close 817ba5bcb98df62e5ed309caecfc1068 2024-11-27T12:26:12,991 DEBUG [RS_CLOSE_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-27T12:26:12,991 DEBUG [RS_CLOSE_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HRegion(1722): Closing 817ba5bcb98df62e5ed309caecfc1068, disabling compactions & flushes 2024-11-27T12:26:12,991 INFO [RS_CLOSE_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068. 2024-11-27T12:26:12,991 DEBUG [RS_CLOSE_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068. 2024-11-27T12:26:12,991 DEBUG [RS_CLOSE_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068. after waiting 0 ms 2024-11-27T12:26:12,991 DEBUG [RS_CLOSE_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068. 2024-11-27T12:26:12,991 INFO [RS_CLOSE_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HRegion(2902): Flushing 817ba5bcb98df62e5ed309caecfc1068 3/3 column families, dataSize=20.13 KB heapSize=53.48 KB 2024-11-27T12:26:12,999 DEBUG [RS_CLOSE_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127ec228808e5664006ab972ed745b38b42_817ba5bcb98df62e5ed309caecfc1068 is 50, key is test_row_0/A:col10/1732710372819/Put/seqid=0 2024-11-27T12:26:13,003 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742260_1436 (size=9914) 2024-11-27T12:26:13,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-27T12:26:13,404 DEBUG [RS_CLOSE_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:26:13,408 INFO [RS_CLOSE_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127ec228808e5664006ab972ed745b38b42_817ba5bcb98df62e5ed309caecfc1068 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127ec228808e5664006ab972ed745b38b42_817ba5bcb98df62e5ed309caecfc1068 2024-11-27T12:26:13,408 DEBUG [RS_CLOSE_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/.tmp/A/d16a7984566d48cfae004c27aca9a8a4, store: [table=TestAcidGuarantees family=A region=817ba5bcb98df62e5ed309caecfc1068] 2024-11-27T12:26:13,409 DEBUG [RS_CLOSE_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/.tmp/A/d16a7984566d48cfae004c27aca9a8a4 is 175, key is test_row_0/A:col10/1732710372819/Put/seqid=0 2024-11-27T12:26:13,411 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742261_1437 (size=22561) 2024-11-27T12:26:13,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-27T12:26:13,812 INFO [RS_CLOSE_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=379, memsize=6.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/.tmp/A/d16a7984566d48cfae004c27aca9a8a4 2024-11-27T12:26:13,819 DEBUG [RS_CLOSE_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/.tmp/B/e09eddb93f054b048407dba447468fac is 50, key is test_row_0/B:col10/1732710372819/Put/seqid=0 2024-11-27T12:26:13,822 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742262_1438 (size=9857) 2024-11-27T12:26:13,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-27T12:26:14,223 INFO [RS_CLOSE_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=379 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/.tmp/B/e09eddb93f054b048407dba447468fac 2024-11-27T12:26:14,237 DEBUG [RS_CLOSE_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/.tmp/C/14156a61fc4c4fb9acb6eae9e6ec036e is 50, key is test_row_0/C:col10/1732710372819/Put/seqid=0 2024-11-27T12:26:14,241 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742263_1439 (size=9857) 2024-11-27T12:26:14,642 INFO [RS_CLOSE_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=379 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/.tmp/C/14156a61fc4c4fb9acb6eae9e6ec036e 2024-11-27T12:26:14,652 DEBUG [RS_CLOSE_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/.tmp/A/d16a7984566d48cfae004c27aca9a8a4 as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/A/d16a7984566d48cfae004c27aca9a8a4 2024-11-27T12:26:14,656 INFO [RS_CLOSE_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/A/d16a7984566d48cfae004c27aca9a8a4, entries=100, sequenceid=379, filesize=22.0 K 2024-11-27T12:26:14,656 DEBUG [RS_CLOSE_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/.tmp/B/e09eddb93f054b048407dba447468fac as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/B/e09eddb93f054b048407dba447468fac 2024-11-27T12:26:14,660 INFO [RS_CLOSE_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/B/e09eddb93f054b048407dba447468fac, entries=100, sequenceid=379, filesize=9.6 K 2024-11-27T12:26:14,661 DEBUG [RS_CLOSE_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/.tmp/C/14156a61fc4c4fb9acb6eae9e6ec036e as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/C/14156a61fc4c4fb9acb6eae9e6ec036e 2024-11-27T12:26:14,664 INFO [RS_CLOSE_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/C/14156a61fc4c4fb9acb6eae9e6ec036e, entries=100, sequenceid=379, filesize=9.6 K 2024-11-27T12:26:14,665 INFO [RS_CLOSE_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HRegion(3140): Finished flush of dataSize ~20.13 KB/20610, heapSize ~53.44 KB/54720, currentSize=0 B/0 for 817ba5bcb98df62e5ed309caecfc1068 in 1674ms, sequenceid=379, compaction requested=true 2024-11-27T12:26:14,665 DEBUG [StoreCloser-TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/A/2f31f74e65114a999211e4044e73716d, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/A/db9f7012f54a4303808bd43b9db6ee6d, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/A/ebe05d131b5049db8bb71024445641f4, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/A/e122c130b1fb4c4fa48f78225470de36] to archive 2024-11-27T12:26:14,666 DEBUG [StoreCloser-TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-27T12:26:14,667 DEBUG [StoreCloser-TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/A/2f31f74e65114a999211e4044e73716d to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/A/2f31f74e65114a999211e4044e73716d 2024-11-27T12:26:14,668 DEBUG [StoreCloser-TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/A/db9f7012f54a4303808bd43b9db6ee6d to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/A/db9f7012f54a4303808bd43b9db6ee6d 2024-11-27T12:26:14,669 DEBUG [StoreCloser-TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/A/ebe05d131b5049db8bb71024445641f4 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/A/ebe05d131b5049db8bb71024445641f4 2024-11-27T12:26:14,670 DEBUG [StoreCloser-TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/A/e122c130b1fb4c4fa48f78225470de36 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/A/e122c130b1fb4c4fa48f78225470de36 2024-11-27T12:26:14,671 DEBUG [StoreCloser-TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/B/d326c753aa0b4516a31dae3ed111329b, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/B/ea98b8357f7943a0ae0441fa0a5f3ace, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/B/c9018cd5ab984fe7b137d0fb4ce1e5c7, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/B/6f4e91191ac149f285429acf222a0f0c] to archive 2024-11-27T12:26:14,671 DEBUG [StoreCloser-TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-27T12:26:14,672 DEBUG [StoreCloser-TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/B/d326c753aa0b4516a31dae3ed111329b to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/B/d326c753aa0b4516a31dae3ed111329b 2024-11-27T12:26:14,673 DEBUG [StoreCloser-TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/B/ea98b8357f7943a0ae0441fa0a5f3ace to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/B/ea98b8357f7943a0ae0441fa0a5f3ace 2024-11-27T12:26:14,674 DEBUG [StoreCloser-TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/B/c9018cd5ab984fe7b137d0fb4ce1e5c7 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/B/c9018cd5ab984fe7b137d0fb4ce1e5c7 2024-11-27T12:26:14,675 DEBUG [StoreCloser-TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/B/6f4e91191ac149f285429acf222a0f0c to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/B/6f4e91191ac149f285429acf222a0f0c 2024-11-27T12:26:14,676 DEBUG [StoreCloser-TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/C/bf26aceda510431db63543ece0a7bfc9, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/C/653ebf67dca34f758a03e73ee82f2e28, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/C/970d3b51cf434dcb9d02151914cac5d2, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/C/64b3a3761b064847b38d7030fbcf4f22] to archive 2024-11-27T12:26:14,676 DEBUG [StoreCloser-TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-27T12:26:14,677 DEBUG [StoreCloser-TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/C/bf26aceda510431db63543ece0a7bfc9 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/C/bf26aceda510431db63543ece0a7bfc9 2024-11-27T12:26:14,678 DEBUG [StoreCloser-TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/C/653ebf67dca34f758a03e73ee82f2e28 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/C/653ebf67dca34f758a03e73ee82f2e28 2024-11-27T12:26:14,679 DEBUG [StoreCloser-TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/C/970d3b51cf434dcb9d02151914cac5d2 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/C/970d3b51cf434dcb9d02151914cac5d2 2024-11-27T12:26:14,680 DEBUG [StoreCloser-TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/C/64b3a3761b064847b38d7030fbcf4f22 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/C/64b3a3761b064847b38d7030fbcf4f22 2024-11-27T12:26:14,684 DEBUG [RS_CLOSE_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/recovered.edits/382.seqid, newMaxSeqId=382, maxSeqId=4 2024-11-27T12:26:14,684 INFO [RS_CLOSE_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068. 2024-11-27T12:26:14,684 DEBUG [RS_CLOSE_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HRegion(1676): Region close journal for 817ba5bcb98df62e5ed309caecfc1068: Waiting for close lock at 1732710372991Running coprocessor pre-close hooks at 1732710372991Disabling compacts and flushes for region at 1732710372991Disabling writes for close at 1732710372991Obtaining lock to block concurrent updates at 1732710372991Preparing flush snapshotting stores in 817ba5bcb98df62e5ed309caecfc1068 at 1732710372991Finished memstore snapshotting TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068., syncing WAL and waiting on mvcc, flushsize=dataSize=20610, getHeapSize=54720, getOffHeapSize=0, getCellsCount=450 at 1732710372992 (+1 ms)Flushing stores of TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068. at 1732710372992Flushing 817ba5bcb98df62e5ed309caecfc1068/A: creating writer at 1732710372992Flushing mob file 817ba5bcb98df62e5ed309caecfc1068/A: appending metadata at 1732710372998 (+6 ms)Flushing mob file 817ba5bcb98df62e5ed309caecfc1068/A: closing flushed file at 1732710372998Flushing 817ba5bcb98df62e5ed309caecfc1068/A: appending metadata at 1732710373408 (+410 ms)Flushing 817ba5bcb98df62e5ed309caecfc1068/A: closing flushed file at 1732710373408Flushing 817ba5bcb98df62e5ed309caecfc1068/B: creating writer at 1732710373816 (+408 ms)Flushing 817ba5bcb98df62e5ed309caecfc1068/B: appending metadata at 1732710373818 (+2 ms)Flushing 817ba5bcb98df62e5ed309caecfc1068/B: closing flushed file at 1732710373818Flushing 817ba5bcb98df62e5ed309caecfc1068/C: creating writer at 1732710374231 (+413 ms)Flushing 817ba5bcb98df62e5ed309caecfc1068/C: appending metadata at 1732710374236 (+5 ms)Flushing 817ba5bcb98df62e5ed309caecfc1068/C: closing flushed file at 1732710374236Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2d04f0b8: reopening flushed file at 1732710374651 (+415 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@56642479: reopening flushed file at 1732710374656 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@412a390a: reopening flushed file at 1732710374661 (+5 ms)Finished flush of dataSize ~20.13 KB/20610, heapSize ~53.44 KB/54720, currentSize=0 B/0 for 817ba5bcb98df62e5ed309caecfc1068 in 1674ms, sequenceid=379, compaction requested=true at 1732710374665 (+4 ms)Writing region close event to WAL at 1732710374681 (+16 ms)Running coprocessor post-close hooks at 1732710374684 (+3 ms)Closed at 1732710374684 2024-11-27T12:26:14,686 INFO [RS_CLOSE_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] handler.UnassignRegionHandler(157): Closed 817ba5bcb98df62e5ed309caecfc1068 2024-11-27T12:26:14,687 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=117 updating hbase:meta row=817ba5bcb98df62e5ed309caecfc1068, regionState=CLOSED 2024-11-27T12:26:14,688 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=118, ppid=117, state=RUNNABLE, hasLock=false; CloseRegionProcedure 817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 because future has completed 2024-11-27T12:26:14,690 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=118, resume processing ppid=117 2024-11-27T12:26:14,690 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=118, ppid=117, state=SUCCESS, hasLock=false; CloseRegionProcedure 817ba5bcb98df62e5ed309caecfc1068, server=1ba5af43fd99,36251,1732710241631 in 1.8510 sec 2024-11-27T12:26:14,693 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=117, resume processing ppid=116 2024-11-27T12:26:14,693 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=117, ppid=116, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=817ba5bcb98df62e5ed309caecfc1068, UNASSIGN in 1.8550 sec 2024-11-27T12:26:14,696 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=116, resume processing ppid=115 2024-11-27T12:26:14,696 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=116, ppid=115, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.8600 sec 2024-11-27T12:26:14,697 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732710374697"}]},"ts":"1732710374697"} 2024-11-27T12:26:14,699 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-27T12:26:14,699 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-27T12:26:14,702 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=115, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.8730 sec 2024-11-27T12:26:14,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-27T12:26:14,979 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-27T12:26:14,980 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-11-27T12:26:14,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] procedure2.ProcedureExecutor(1139): Stored pid=119, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-27T12:26:14,984 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=119, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-27T12:26:14,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-27T12:26:14,986 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=119, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-27T12:26:14,990 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068 2024-11-27T12:26:14,993 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/A, FileablePath, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/B, FileablePath, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/C, FileablePath, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/recovered.edits] 2024-11-27T12:26:14,997 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/A/19f4dc00ecc4454d98bde82fdc959a51 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/A/19f4dc00ecc4454d98bde82fdc959a51 2024-11-27T12:26:14,998 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/A/4cda6e8e585143739e4b4c0e51f69b5e to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/A/4cda6e8e585143739e4b4c0e51f69b5e 2024-11-27T12:26:14,999 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/A/d16a7984566d48cfae004c27aca9a8a4 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/A/d16a7984566d48cfae004c27aca9a8a4 2024-11-27T12:26:15,003 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/B/0413aa05788d42859bad57ec2df125f1 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/B/0413aa05788d42859bad57ec2df125f1 2024-11-27T12:26:15,004 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/B/5fae9d0aef344578adfdb661f96e7ced to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/B/5fae9d0aef344578adfdb661f96e7ced 2024-11-27T12:26:15,007 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/B/e09eddb93f054b048407dba447468fac to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/B/e09eddb93f054b048407dba447468fac 2024-11-27T12:26:15,009 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/C/14156a61fc4c4fb9acb6eae9e6ec036e to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/C/14156a61fc4c4fb9acb6eae9e6ec036e 2024-11-27T12:26:15,010 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/C/a5ee312d754e4fe490f95dee90ed121c to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/C/a5ee312d754e4fe490f95dee90ed121c 2024-11-27T12:26:15,011 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/C/fa9e0e5b230143a2af0eab553aa6f756 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/C/fa9e0e5b230143a2af0eab553aa6f756 2024-11-27T12:26:15,015 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/recovered.edits/382.seqid to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068/recovered.edits/382.seqid 2024-11-27T12:26:15,015 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/817ba5bcb98df62e5ed309caecfc1068 2024-11-27T12:26:15,015 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-27T12:26:15,016 DEBUG [PEWorker-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-27T12:26:15,016 DEBUG [PEWorker-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-27T12:26:15,019 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127166ec1798f354071b9b8974964ec0efc_817ba5bcb98df62e5ed309caecfc1068 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127166ec1798f354071b9b8974964ec0efc_817ba5bcb98df62e5ed309caecfc1068 2024-11-27T12:26:15,020 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411274301c71aa7f94da98cf3e933ba9ff650_817ba5bcb98df62e5ed309caecfc1068 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411274301c71aa7f94da98cf3e933ba9ff650_817ba5bcb98df62e5ed309caecfc1068 2024-11-27T12:26:15,020 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112759c5f5d5da8f4cb095cb37a80101f03d_817ba5bcb98df62e5ed309caecfc1068 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112759c5f5d5da8f4cb095cb37a80101f03d_817ba5bcb98df62e5ed309caecfc1068 2024-11-27T12:26:15,021 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411275b22198619fd464bb94843bb6cc3668d_817ba5bcb98df62e5ed309caecfc1068 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411275b22198619fd464bb94843bb6cc3668d_817ba5bcb98df62e5ed309caecfc1068 2024-11-27T12:26:15,022 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411275fda1fa758bf498294368c4df1ceaf3f_817ba5bcb98df62e5ed309caecfc1068 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411275fda1fa758bf498294368c4df1ceaf3f_817ba5bcb98df62e5ed309caecfc1068 2024-11-27T12:26:15,023 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127604389cd26114820869c01a67652df5f_817ba5bcb98df62e5ed309caecfc1068 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127604389cd26114820869c01a67652df5f_817ba5bcb98df62e5ed309caecfc1068 2024-11-27T12:26:15,023 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411276da5353fb6e04c649432ae38e0a14e45_817ba5bcb98df62e5ed309caecfc1068 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411276da5353fb6e04c649432ae38e0a14e45_817ba5bcb98df62e5ed309caecfc1068 2024-11-27T12:26:15,024 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411277137b0c26e644c0daf6ae44dd6b9c7e7_817ba5bcb98df62e5ed309caecfc1068 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411277137b0c26e644c0daf6ae44dd6b9c7e7_817ba5bcb98df62e5ed309caecfc1068 2024-11-27T12:26:15,025 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411278411adc70526415d8fc3f16e08e5c20b_817ba5bcb98df62e5ed309caecfc1068 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411278411adc70526415d8fc3f16e08e5c20b_817ba5bcb98df62e5ed309caecfc1068 2024-11-27T12:26:15,025 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411278aae8f7a3cc14bd69ea2c3e0ef847563_817ba5bcb98df62e5ed309caecfc1068 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411278aae8f7a3cc14bd69ea2c3e0ef847563_817ba5bcb98df62e5ed309caecfc1068 2024-11-27T12:26:15,026 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411278ae504ae0aff43d1a8aae0239376e5c5_817ba5bcb98df62e5ed309caecfc1068 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411278ae504ae0aff43d1a8aae0239376e5c5_817ba5bcb98df62e5ed309caecfc1068 2024-11-27T12:26:15,027 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127a33f5304d2134290b04e23080d579125_817ba5bcb98df62e5ed309caecfc1068 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127a33f5304d2134290b04e23080d579125_817ba5bcb98df62e5ed309caecfc1068 2024-11-27T12:26:15,027 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127a96b95dfbde54db3be06f150ec4c1412_817ba5bcb98df62e5ed309caecfc1068 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127a96b95dfbde54db3be06f150ec4c1412_817ba5bcb98df62e5ed309caecfc1068 2024-11-27T12:26:15,028 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127b0663af5fb11486e8d0019189b9f754d_817ba5bcb98df62e5ed309caecfc1068 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127b0663af5fb11486e8d0019189b9f754d_817ba5bcb98df62e5ed309caecfc1068 2024-11-27T12:26:15,029 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127d45b03a57566468cac57ab494ce0d378_817ba5bcb98df62e5ed309caecfc1068 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127d45b03a57566468cac57ab494ce0d378_817ba5bcb98df62e5ed309caecfc1068 2024-11-27T12:26:15,029 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127e333e2e8f01a403aa9ffd9baadbfb143_817ba5bcb98df62e5ed309caecfc1068 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127e333e2e8f01a403aa9ffd9baadbfb143_817ba5bcb98df62e5ed309caecfc1068 2024-11-27T12:26:15,030 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127e80b5daef4e94291b6fb031c0cabd27b_817ba5bcb98df62e5ed309caecfc1068 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127e80b5daef4e94291b6fb031c0cabd27b_817ba5bcb98df62e5ed309caecfc1068 2024-11-27T12:26:15,031 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127ec228808e5664006ab972ed745b38b42_817ba5bcb98df62e5ed309caecfc1068 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127ec228808e5664006ab972ed745b38b42_817ba5bcb98df62e5ed309caecfc1068 2024-11-27T12:26:15,031 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127ecd6f9601855454bb8998460c810df21_817ba5bcb98df62e5ed309caecfc1068 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127ecd6f9601855454bb8998460c810df21_817ba5bcb98df62e5ed309caecfc1068 2024-11-27T12:26:15,032 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127f250a53b2beb41849231825735062507_817ba5bcb98df62e5ed309caecfc1068 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127f250a53b2beb41849231825735062507_817ba5bcb98df62e5ed309caecfc1068 2024-11-27T12:26:15,033 DEBUG [PEWorker-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-27T12:26:15,034 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=119, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-27T12:26:15,036 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-27T12:26:15,037 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-27T12:26:15,038 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=119, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-27T12:26:15,038 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-27T12:26:15,038 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732710375038"}]},"ts":"9223372036854775807"} 2024-11-27T12:26:15,040 INFO [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-27T12:26:15,040 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 817ba5bcb98df62e5ed309caecfc1068, NAME => 'TestAcidGuarantees,,1732710341445.817ba5bcb98df62e5ed309caecfc1068.', STARTKEY => '', ENDKEY => ''}] 2024-11-27T12:26:15,040 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-27T12:26:15,040 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732710375040"}]},"ts":"9223372036854775807"} 2024-11-27T12:26:15,042 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-27T12:26:15,042 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(133): Finished pid=119, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-27T12:26:15,043 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=119, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 62 msec 2024-11-27T12:26:15,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-27T12:26:15,088 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-27T12:26:15,088 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-27T12:26:15,099 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMobScanAtomicity Thread=239 (was 240), OpenFileDescriptor=443 (was 453), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=464 (was 457) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=8669 (was 8615) - AvailableMemoryMB LEAK? - 2024-11-27T12:26:15,109 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testScanAtomicity Thread=239, OpenFileDescriptor=443, MaxFileDescriptor=1048576, SystemLoadAverage=464, ProcessCount=11, AvailableMemoryMB=8669 2024-11-27T12:26:15,110 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-27T12:26:15,111 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-27T12:26:15,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] procedure2.ProcedureExecutor(1139): Stored pid=120, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-27T12:26:15,113 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=120, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-27T12:26:15,113 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:26:15,113 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 120 2024-11-27T12:26:15,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=120 2024-11-27T12:26:15,114 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=120, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-27T12:26:15,118 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742264_1440 (size=959) 2024-11-27T12:26:15,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=120 2024-11-27T12:26:15,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=120 2024-11-27T12:26:15,520 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 3144c5c576970df3a2a1c91af06cc315, NAME => 'TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c 2024-11-27T12:26:15,527 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742265_1441 (size=53) 2024-11-27T12:26:15,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=120 2024-11-27T12:26:15,929 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T12:26:15,930 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 3144c5c576970df3a2a1c91af06cc315, disabling compactions & flushes 2024-11-27T12:26:15,930 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315. 2024-11-27T12:26:15,930 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315. 2024-11-27T12:26:15,930 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315. after waiting 0 ms 2024-11-27T12:26:15,930 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315. 2024-11-27T12:26:15,930 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315. 2024-11-27T12:26:15,930 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 3144c5c576970df3a2a1c91af06cc315: Waiting for close lock at 1732710375930Disabling compacts and flushes for region at 1732710375930Disabling writes for close at 1732710375930Writing region close event to WAL at 1732710375930Closed at 1732710375930 2024-11-27T12:26:15,933 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=120, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-27T12:26:15,934 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732710375933"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732710375933"}]},"ts":"1732710375933"} 2024-11-27T12:26:15,936 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-27T12:26:15,938 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=120, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-27T12:26:15,938 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732710375938"}]},"ts":"1732710375938"} 2024-11-27T12:26:15,940 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-27T12:26:15,940 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=121, ppid=120, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=3144c5c576970df3a2a1c91af06cc315, ASSIGN}] 2024-11-27T12:26:15,941 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=121, ppid=120, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=3144c5c576970df3a2a1c91af06cc315, ASSIGN 2024-11-27T12:26:15,943 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=121, ppid=120, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=3144c5c576970df3a2a1c91af06cc315, ASSIGN; state=OFFLINE, location=1ba5af43fd99,36251,1732710241631; forceNewPlan=false, retain=false 2024-11-27T12:26:16,093 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=121 updating hbase:meta row=3144c5c576970df3a2a1c91af06cc315, regionState=OPENING, regionLocation=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:16,096 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=121, ppid=120, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=3144c5c576970df3a2a1c91af06cc315, ASSIGN because future has completed 2024-11-27T12:26:16,096 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=122, ppid=121, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631}] 2024-11-27T12:26:16,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=120 2024-11-27T12:26:16,252 INFO [RS_OPEN_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315. 2024-11-27T12:26:16,252 DEBUG [RS_OPEN_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegion(7752): Opening region: {ENCODED => 3144c5c576970df3a2a1c91af06cc315, NAME => 'TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315.', STARTKEY => '', ENDKEY => ''} 2024-11-27T12:26:16,253 DEBUG [RS_OPEN_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 3144c5c576970df3a2a1c91af06cc315 2024-11-27T12:26:16,253 DEBUG [RS_OPEN_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T12:26:16,253 DEBUG [RS_OPEN_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegion(7794): checking encryption for 3144c5c576970df3a2a1c91af06cc315 2024-11-27T12:26:16,253 DEBUG [RS_OPEN_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegion(7797): checking classloading for 3144c5c576970df3a2a1c91af06cc315 2024-11-27T12:26:16,255 INFO [StoreOpener-3144c5c576970df3a2a1c91af06cc315-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 3144c5c576970df3a2a1c91af06cc315 2024-11-27T12:26:16,257 INFO [StoreOpener-3144c5c576970df3a2a1c91af06cc315-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3144c5c576970df3a2a1c91af06cc315 columnFamilyName A 2024-11-27T12:26:16,258 DEBUG [StoreOpener-3144c5c576970df3a2a1c91af06cc315-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:26:16,258 INFO [StoreOpener-3144c5c576970df3a2a1c91af06cc315-1 {}] regionserver.HStore(327): Store=3144c5c576970df3a2a1c91af06cc315/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T12:26:16,259 INFO [StoreOpener-3144c5c576970df3a2a1c91af06cc315-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 3144c5c576970df3a2a1c91af06cc315 2024-11-27T12:26:16,261 INFO [StoreOpener-3144c5c576970df3a2a1c91af06cc315-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3144c5c576970df3a2a1c91af06cc315 columnFamilyName B 2024-11-27T12:26:16,261 DEBUG [StoreOpener-3144c5c576970df3a2a1c91af06cc315-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:26:16,261 INFO [StoreOpener-3144c5c576970df3a2a1c91af06cc315-1 {}] regionserver.HStore(327): Store=3144c5c576970df3a2a1c91af06cc315/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T12:26:16,262 INFO [StoreOpener-3144c5c576970df3a2a1c91af06cc315-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 3144c5c576970df3a2a1c91af06cc315 2024-11-27T12:26:16,263 INFO [StoreOpener-3144c5c576970df3a2a1c91af06cc315-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3144c5c576970df3a2a1c91af06cc315 columnFamilyName C 2024-11-27T12:26:16,264 DEBUG [StoreOpener-3144c5c576970df3a2a1c91af06cc315-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:26:16,264 INFO [StoreOpener-3144c5c576970df3a2a1c91af06cc315-1 {}] regionserver.HStore(327): Store=3144c5c576970df3a2a1c91af06cc315/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T12:26:16,265 DEBUG [RS_OPEN_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegion(1038): replaying wal for 3144c5c576970df3a2a1c91af06cc315 2024-11-27T12:26:16,266 DEBUG [RS_OPEN_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315 2024-11-27T12:26:16,266 DEBUG [RS_OPEN_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315 2024-11-27T12:26:16,268 DEBUG [RS_OPEN_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegion(1048): stopping wal replay for 3144c5c576970df3a2a1c91af06cc315 2024-11-27T12:26:16,268 DEBUG [RS_OPEN_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegion(1060): Cleaning up temporary data for 3144c5c576970df3a2a1c91af06cc315 2024-11-27T12:26:16,269 DEBUG [RS_OPEN_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-27T12:26:16,271 DEBUG [RS_OPEN_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegion(1093): writing seq id for 3144c5c576970df3a2a1c91af06cc315 2024-11-27T12:26:16,274 DEBUG [RS_OPEN_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-27T12:26:16,274 INFO [RS_OPEN_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegion(1114): Opened 3144c5c576970df3a2a1c91af06cc315; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70476886, jitterRate=0.05018743872642517}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-27T12:26:16,274 DEBUG [RS_OPEN_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 3144c5c576970df3a2a1c91af06cc315 2024-11-27T12:26:16,275 DEBUG [RS_OPEN_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegion(1006): Region open journal for 3144c5c576970df3a2a1c91af06cc315: Running coprocessor pre-open hook at 1732710376254Writing region info on filesystem at 1732710376254Initializing all the Stores at 1732710376255 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732710376255Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732710376255Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732710376255Cleaning up temporary data from old regions at 1732710376268 (+13 ms)Running coprocessor post-open hooks at 1732710376274 (+6 ms)Region opened successfully at 1732710376275 (+1 ms) 2024-11-27T12:26:16,276 INFO [RS_OPEN_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., pid=122, masterSystemTime=1732710376248 2024-11-27T12:26:16,278 DEBUG [RS_OPEN_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315. 2024-11-27T12:26:16,278 INFO [RS_OPEN_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315. 2024-11-27T12:26:16,279 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=121 updating hbase:meta row=3144c5c576970df3a2a1c91af06cc315, regionState=OPEN, openSeqNum=2, regionLocation=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:16,281 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=122, ppid=121, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 because future has completed 2024-11-27T12:26:16,285 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=122, resume processing ppid=121 2024-11-27T12:26:16,285 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=122, ppid=121, state=SUCCESS, hasLock=false; OpenRegionProcedure 3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 in 186 msec 2024-11-27T12:26:16,287 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=121, resume processing ppid=120 2024-11-27T12:26:16,287 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=121, ppid=120, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=3144c5c576970df3a2a1c91af06cc315, ASSIGN in 345 msec 2024-11-27T12:26:16,288 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=120, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-27T12:26:16,288 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732710376288"}]},"ts":"1732710376288"} 2024-11-27T12:26:16,290 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-27T12:26:16,291 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=120, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-27T12:26:16,292 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=120, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1800 sec 2024-11-27T12:26:17,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=120 2024-11-27T12:26:17,260 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-27T12:26:17,263 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@47b386e0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T12:26:17,263 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1ba5af43fd99,36215,-1 for getting cluster id 2024-11-27T12:26:17,263 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T12:26:17,266 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '200978c2-2c41-4f93-b0b5-127ea1024f37' 2024-11-27T12:26:17,266 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T12:26:17,266 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "200978c2-2c41-4f93-b0b5-127ea1024f37" 2024-11-27T12:26:17,266 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@36ba9070, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T12:26:17,266 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1ba5af43fd99,36215,-1] 2024-11-27T12:26:17,266 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T12:26:17,267 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T12:26:17,268 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60688, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T12:26:17,268 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@653d5bdd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T12:26:17,269 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T12:26:17,269 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1ba5af43fd99,36251,1732710241631, seqNum=-1] 2024-11-27T12:26:17,270 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T12:26:17,271 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39662, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T12:26:17,271 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-27T12:26:17,272 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncConnectionImpl(321): The fetched master address is 1ba5af43fd99,36215,1732710240860 2024-11-27T12:26:17,272 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@7ef3ff8a 2024-11-27T12:26:17,272 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-27T12:26:17,273 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60696, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-27T12:26:17,275 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@621cfad6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T12:26:17,275 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1ba5af43fd99,36215,-1 for getting cluster id 2024-11-27T12:26:17,275 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T12:26:17,276 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '200978c2-2c41-4f93-b0b5-127ea1024f37' 2024-11-27T12:26:17,276 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T12:26:17,276 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "200978c2-2c41-4f93-b0b5-127ea1024f37" 2024-11-27T12:26:17,276 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2caa908c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T12:26:17,276 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1ba5af43fd99,36215,-1] 2024-11-27T12:26:17,276 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T12:26:17,276 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T12:26:17,277 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60700, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T12:26:17,278 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@55a84114, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T12:26:17,279 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@27842489, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T12:26:17,279 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1ba5af43fd99,36215,-1 for getting cluster id 2024-11-27T12:26:17,279 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T12:26:17,279 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '200978c2-2c41-4f93-b0b5-127ea1024f37' 2024-11-27T12:26:17,280 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T12:26:17,280 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "200978c2-2c41-4f93-b0b5-127ea1024f37" 2024-11-27T12:26:17,280 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@56581693, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T12:26:17,280 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1ba5af43fd99,36215,-1] 2024-11-27T12:26:17,280 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T12:26:17,280 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T12:26:17,281 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60712, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T12:26:17,281 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@16160631, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T12:26:17,282 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3464d23e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T12:26:17,282 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1ba5af43fd99,36215,-1 for getting cluster id 2024-11-27T12:26:17,282 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T12:26:17,283 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '200978c2-2c41-4f93-b0b5-127ea1024f37' 2024-11-27T12:26:17,283 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T12:26:17,283 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "200978c2-2c41-4f93-b0b5-127ea1024f37" 2024-11-27T12:26:17,283 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6e6c467e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T12:26:17,283 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1ba5af43fd99,36215,-1] 2024-11-27T12:26:17,284 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T12:26:17,284 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T12:26:17,284 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60722, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T12:26:17,285 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@c4dbdfe, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T12:26:17,286 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@fe65078, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T12:26:17,286 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1ba5af43fd99,36215,-1 for getting cluster id 2024-11-27T12:26:17,286 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T12:26:17,287 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '200978c2-2c41-4f93-b0b5-127ea1024f37' 2024-11-27T12:26:17,287 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T12:26:17,287 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "200978c2-2c41-4f93-b0b5-127ea1024f37" 2024-11-27T12:26:17,287 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2aeb97c6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T12:26:17,287 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1ba5af43fd99,36215,-1] 2024-11-27T12:26:17,287 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T12:26:17,288 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T12:26:17,288 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60736, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T12:26:17,289 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3869e7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T12:26:17,290 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3688e60, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T12:26:17,290 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1ba5af43fd99,36215,-1 for getting cluster id 2024-11-27T12:26:17,290 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T12:26:17,291 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '200978c2-2c41-4f93-b0b5-127ea1024f37' 2024-11-27T12:26:17,291 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T12:26:17,291 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "200978c2-2c41-4f93-b0b5-127ea1024f37" 2024-11-27T12:26:17,291 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@444c6000, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T12:26:17,291 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1ba5af43fd99,36215,-1] 2024-11-27T12:26:17,292 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T12:26:17,292 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T12:26:17,293 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60742, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T12:26:17,293 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@66d24583, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T12:26:17,294 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1293f9dd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T12:26:17,294 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1ba5af43fd99,36215,-1 for getting cluster id 2024-11-27T12:26:17,294 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T12:26:17,295 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '200978c2-2c41-4f93-b0b5-127ea1024f37' 2024-11-27T12:26:17,295 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T12:26:17,295 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "200978c2-2c41-4f93-b0b5-127ea1024f37" 2024-11-27T12:26:17,295 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19703b2b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T12:26:17,295 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1ba5af43fd99,36215,-1] 2024-11-27T12:26:17,296 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T12:26:17,296 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T12:26:17,296 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60754, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T12:26:17,297 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@a07d810, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T12:26:17,298 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7bffa8c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T12:26:17,298 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1ba5af43fd99,36215,-1 for getting cluster id 2024-11-27T12:26:17,298 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T12:26:17,299 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '200978c2-2c41-4f93-b0b5-127ea1024f37' 2024-11-27T12:26:17,299 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T12:26:17,299 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "200978c2-2c41-4f93-b0b5-127ea1024f37" 2024-11-27T12:26:17,299 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@d483c4f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T12:26:17,299 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1ba5af43fd99,36215,-1] 2024-11-27T12:26:17,299 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T12:26:17,299 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T12:26:17,300 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60770, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T12:26:17,301 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6c2952e0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T12:26:17,301 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2c22415d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T12:26:17,302 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1ba5af43fd99,36215,-1 for getting cluster id 2024-11-27T12:26:17,302 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T12:26:17,302 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '200978c2-2c41-4f93-b0b5-127ea1024f37' 2024-11-27T12:26:17,303 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T12:26:17,303 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "200978c2-2c41-4f93-b0b5-127ea1024f37" 2024-11-27T12:26:17,303 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@17a131c9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T12:26:17,303 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1ba5af43fd99,36215,-1] 2024-11-27T12:26:17,303 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T12:26:17,303 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T12:26:17,304 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60786, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T12:26:17,304 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d4685b7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T12:26:17,305 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3bce704e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T12:26:17,305 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1ba5af43fd99,36215,-1 for getting cluster id 2024-11-27T12:26:17,305 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T12:26:17,306 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '200978c2-2c41-4f93-b0b5-127ea1024f37' 2024-11-27T12:26:17,306 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T12:26:17,306 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "200978c2-2c41-4f93-b0b5-127ea1024f37" 2024-11-27T12:26:17,306 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d969f45, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T12:26:17,306 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1ba5af43fd99,36215,-1] 2024-11-27T12:26:17,307 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T12:26:17,307 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T12:26:17,308 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60792, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T12:26:17,308 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1df73511, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T12:26:17,309 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6fc5b25c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T12:26:17,309 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1ba5af43fd99,36215,-1 for getting cluster id 2024-11-27T12:26:17,309 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T12:26:17,310 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '200978c2-2c41-4f93-b0b5-127ea1024f37' 2024-11-27T12:26:17,310 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T12:26:17,310 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "200978c2-2c41-4f93-b0b5-127ea1024f37" 2024-11-27T12:26:17,311 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@54e75710, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T12:26:17,311 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1ba5af43fd99,36215,-1] 2024-11-27T12:26:17,311 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T12:26:17,311 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T12:26:17,312 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60814, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T12:26:17,313 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@30d8dc79, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T12:26:17,313 DEBUG [Thread-1993 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T12:26:17,313 DEBUG [Thread-1999 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T12:26:17,313 DEBUG [Thread-1990 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T12:26:17,314 DEBUG [Thread-2001 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T12:26:17,314 DEBUG [Thread-1984 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T12:26:17,314 DEBUG [Thread-1986 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T12:26:17,314 DEBUG [Thread-1995 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T12:26:17,314 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1ba5af43fd99,36251,1732710241631, seqNum=-1] 2024-11-27T12:26:17,314 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1ba5af43fd99,36251,1732710241631, seqNum=-1] 2024-11-27T12:26:17,315 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T12:26:17,315 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T12:26:17,315 DEBUG [Thread-1988 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T12:26:17,315 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-27T12:26:17,315 DEBUG [Thread-1982 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T12:26:17,315 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1ba5af43fd99,36251,1732710241631, seqNum=-1] 2024-11-27T12:26:17,315 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T12:26:17,315 DEBUG [Thread-1995 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1ba5af43fd99,36251,1732710241631, seqNum=-1] 2024-11-27T12:26:17,315 DEBUG [Thread-1997 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T12:26:17,316 DEBUG [Thread-1995 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T12:26:17,316 DEBUG [Thread-1984 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1ba5af43fd99,36251,1732710241631, seqNum=-1] 2024-11-27T12:26:17,316 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1ba5af43fd99,36251,1732710241631, seqNum=-1] 2024-11-27T12:26:17,316 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39674, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T12:26:17,316 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39672, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T12:26:17,316 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1ba5af43fd99,36251,1732710241631, seqNum=-1] 2024-11-27T12:26:17,316 DEBUG [Thread-1984 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T12:26:17,317 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T12:26:17,317 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T12:26:17,317 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1ba5af43fd99,36251,1732710241631, seqNum=-1] 2024-11-27T12:26:17,317 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1ba5af43fd99,36251,1732710241631, seqNum=-1] 2024-11-27T12:26:17,317 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T12:26:17,317 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T12:26:17,317 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39682, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T12:26:17,318 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39696, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T12:26:17,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] procedure2.ProcedureExecutor(1139): Stored pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees 2024-11-27T12:26:17,318 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39698, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T12:26:17,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-27T12:26:17,319 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39716, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T12:26:17,319 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39710, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T12:26:17,319 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2] 2024-11-27T12:26:17,319 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39712, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T12:26:17,319 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2] 2024-11-27T12:26:17,319 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2] 2024-11-27T12:26:17,319 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2] 2024-11-27T12:26:17,319 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T12:26:17,320 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39680, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T12:26:17,321 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1ba5af43fd99,36251,1732710241631, seqNum=-1] 2024-11-27T12:26:17,321 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2] 2024-11-27T12:26:17,321 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T12:26:17,321 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2] 2024-11-27T12:26:17,321 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2] 2024-11-27T12:26:17,322 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2] 2024-11-27T12:26:17,322 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T12:26:17,322 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=124, ppid=123, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T12:26:17,322 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2] 2024-11-27T12:26:17,323 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39730, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T12:26:17,324 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2] 2024-11-27T12:26:17,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(8855): Flush requested on 3144c5c576970df3a2a1c91af06cc315 2024-11-27T12:26:17,327 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T12:26:17,327 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3144c5c576970df3a2a1c91af06cc315 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-27T12:26:17,354 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/.tmp/A/b1a17dce207041728e6a4cfa67964ffa is 50, key is test_row_0/A:col10/1732710377327/Put/seqid=0 2024-11-27T12:26:17,358 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742266_1442 (size=16681) 2024-11-27T12:26:17,358 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/.tmp/A/b1a17dce207041728e6a4cfa67964ffa 2024-11-27T12:26:17,363 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:17,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39674 deadline: 1732710437358, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:17,363 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:17,363 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:17,363 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:26:17,370 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:17,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39710 deadline: 1732710437360, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:17,370 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:17,370 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:17,370 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:17,370 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:26:17,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39698 deadline: 1732710437361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:17,370 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:17,370 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:17,371 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:17,371 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:26:17,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39712 deadline: 1732710437361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:17,371 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:17,371 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:17,371 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:26:17,373 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:17,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39696 deadline: 1732710437363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:17,373 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:17,373 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:17,373 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:26:17,381 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/.tmp/B/eb948be447f54b5eb6b2b773996a6e84 is 50, key is test_row_0/B:col10/1732710377327/Put/seqid=0 2024-11-27T12:26:17,385 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742267_1443 (size=12001) 2024-11-27T12:26:17,385 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/.tmp/B/eb948be447f54b5eb6b2b773996a6e84 2024-11-27T12:26:17,404 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/.tmp/C/16b0a48d92ae45c5900bdde6eaecfdb2 is 50, key is test_row_0/C:col10/1732710377327/Put/seqid=0 2024-11-27T12:26:17,408 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742268_1444 (size=12001) 2024-11-27T12:26:17,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-27T12:26:17,473 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:17,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39674 deadline: 1732710437467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:17,473 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:17,473 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:17,473 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:26:17,476 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-27T12:26:17,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315. 2024-11-27T12:26:17,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315. as already flushing 2024-11-27T12:26:17,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315. 2024-11-27T12:26:17,476 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 3144c5c576970df3a2a1c91af06cc315, NAME => 'TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:26:17,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 3144c5c576970df3a2a1c91af06cc315, NAME => 'TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:26:17,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3144c5c576970df3a2a1c91af06cc315, NAME => 'TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3144c5c576970df3a2a1c91af06cc315, NAME => 'TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:26:17,484 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:17,484 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:17,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39710 deadline: 1732710437478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:17,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39698 deadline: 1732710437478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:17,484 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:17,484 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:17,484 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:17,484 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:26:17,484 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:17,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39712 deadline: 1732710437478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:17,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39696 deadline: 1732710437478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:17,484 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:17,484 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:17,484 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:26:17,484 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:17,484 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:17,484 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:26:17,484 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:17,484 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:17,484 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:26:17,629 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-27T12:26:17,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315. 2024-11-27T12:26:17,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315. as already flushing 2024-11-27T12:26:17,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315. 2024-11-27T12:26:17,629 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 3144c5c576970df3a2a1c91af06cc315, NAME => 'TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:26:17,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 3144c5c576970df3a2a1c91af06cc315, NAME => 'TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:26:17,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3144c5c576970df3a2a1c91af06cc315, NAME => 'TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3144c5c576970df3a2a1c91af06cc315, NAME => 'TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:26:17,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-27T12:26:17,680 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:17,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39674 deadline: 1732710437678, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:17,681 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:17,681 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:17,681 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:26:17,693 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:17,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39710 deadline: 1732710437688, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:17,693 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:17,693 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:17,694 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:17,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39698 deadline: 1732710437688, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:17,694 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:26:17,694 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:17,694 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:17,694 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:17,694 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:17,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39696 deadline: 1732710437688, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:17,694 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:26:17,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39712 deadline: 1732710437688, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:17,694 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:17,694 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:17,694 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:17,694 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:26:17,694 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:17,694 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:26:17,782 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-27T12:26:17,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315. 2024-11-27T12:26:17,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315. as already flushing 2024-11-27T12:26:17,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315. 2024-11-27T12:26:17,782 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 3144c5c576970df3a2a1c91af06cc315, NAME => 'TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:26:17,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 3144c5c576970df3a2a1c91af06cc315, NAME => 'TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:26:17,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3144c5c576970df3a2a1c91af06cc315, NAME => 'TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3144c5c576970df3a2a1c91af06cc315, NAME => 'TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:26:17,808 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/.tmp/C/16b0a48d92ae45c5900bdde6eaecfdb2 2024-11-27T12:26:17,814 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/.tmp/A/b1a17dce207041728e6a4cfa67964ffa as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/A/b1a17dce207041728e6a4cfa67964ffa 2024-11-27T12:26:17,816 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/A/b1a17dce207041728e6a4cfa67964ffa, entries=250, sequenceid=12, filesize=16.3 K 2024-11-27T12:26:17,817 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/.tmp/B/eb948be447f54b5eb6b2b773996a6e84 as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/B/eb948be447f54b5eb6b2b773996a6e84 2024-11-27T12:26:17,820 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/B/eb948be447f54b5eb6b2b773996a6e84, entries=150, sequenceid=12, filesize=11.7 K 2024-11-27T12:26:17,820 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/.tmp/C/16b0a48d92ae45c5900bdde6eaecfdb2 as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/C/16b0a48d92ae45c5900bdde6eaecfdb2 2024-11-27T12:26:17,825 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/C/16b0a48d92ae45c5900bdde6eaecfdb2, entries=150, sequenceid=12, filesize=11.7 K 2024-11-27T12:26:17,826 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 3144c5c576970df3a2a1c91af06cc315 in 499ms, sequenceid=12, compaction requested=false 2024-11-27T12:26:17,826 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3144c5c576970df3a2a1c91af06cc315: 2024-11-27T12:26:17,934 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-27T12:26:17,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315. 2024-11-27T12:26:17,934 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2902): Flushing 3144c5c576970df3a2a1c91af06cc315 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-27T12:26:17,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/.tmp/A/8663aa07aae84174a946123426cd00bf is 50, key is test_row_0/A:col10/1732710377360/Put/seqid=0 2024-11-27T12:26:17,943 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742269_1445 (size=12001) 2024-11-27T12:26:17,944 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/.tmp/A/8663aa07aae84174a946123426cd00bf 2024-11-27T12:26:17,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-27T12:26:17,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/.tmp/B/1eaa295f51194d919d5c4a218f596055 is 50, key is test_row_0/B:col10/1732710377360/Put/seqid=0 2024-11-27T12:26:17,955 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742270_1446 (size=12001) 2024-11-27T12:26:17,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(8855): Flush requested on 3144c5c576970df3a2a1c91af06cc315 2024-11-27T12:26:17,990 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315. as already flushing 2024-11-27T12:26:18,008 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:18,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39696 deadline: 1732710438003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:18,008 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:18,008 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:18,008 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:26:18,012 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:18,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39712 deadline: 1732710438004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:18,012 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:18,012 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:18,012 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:18,012 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:26:18,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39710 deadline: 1732710438005, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:18,012 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:18,012 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:18,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39674 deadline: 1732710438008, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:18,012 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:18,012 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:26:18,012 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:18,013 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:18,013 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:18,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39698 deadline: 1732710438008, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:18,013 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:26:18,013 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:18,013 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:18,013 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:26:18,120 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:18,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39696 deadline: 1732710438118, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:18,120 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:18,120 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:18,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39712 deadline: 1732710438118, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:18,120 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:18,120 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:26:18,120 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:18,120 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:18,120 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:18,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39710 deadline: 1732710438118, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:18,120 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:26:18,121 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:18,121 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:18,121 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:26:18,124 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:18,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39698 deadline: 1732710438119, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:18,124 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:18,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39674 deadline: 1732710438119, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:18,124 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:18,124 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:18,124 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:26:18,124 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:18,124 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:18,124 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:26:18,330 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:18,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39696 deadline: 1732710438328, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:18,330 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:18,330 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:18,330 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:18,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39712 deadline: 1732710438328, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:18,330 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:26:18,331 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:18,331 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:18,331 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:18,331 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:26:18,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39710 deadline: 1732710438328, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:18,331 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:18,331 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:18,331 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:18,331 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:26:18,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39698 deadline: 1732710438329, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:18,331 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:18,331 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:18,331 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:18,331 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:26:18,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39674 deadline: 1732710438329, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:18,332 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:18,332 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:18,332 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:26:18,356 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/.tmp/B/1eaa295f51194d919d5c4a218f596055 2024-11-27T12:26:18,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/.tmp/C/ae5edde338334cd4a72f5a131da0389c is 50, key is test_row_0/C:col10/1732710377360/Put/seqid=0 2024-11-27T12:26:18,374 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742271_1447 (size=12001) 2024-11-27T12:26:18,380 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/.tmp/C/ae5edde338334cd4a72f5a131da0389c 2024-11-27T12:26:18,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/.tmp/A/8663aa07aae84174a946123426cd00bf as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/A/8663aa07aae84174a946123426cd00bf 2024-11-27T12:26:18,389 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/A/8663aa07aae84174a946123426cd00bf, entries=150, sequenceid=37, filesize=11.7 K 2024-11-27T12:26:18,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/.tmp/B/1eaa295f51194d919d5c4a218f596055 as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/B/1eaa295f51194d919d5c4a218f596055 2024-11-27T12:26:18,397 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/B/1eaa295f51194d919d5c4a218f596055, entries=150, sequenceid=37, filesize=11.7 K 2024-11-27T12:26:18,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/.tmp/C/ae5edde338334cd4a72f5a131da0389c as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/C/ae5edde338334cd4a72f5a131da0389c 2024-11-27T12:26:18,403 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/C/ae5edde338334cd4a72f5a131da0389c, entries=150, sequenceid=37, filesize=11.7 K 2024-11-27T12:26:18,404 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=60.38 KB/61830 for 3144c5c576970df3a2a1c91af06cc315 in 470ms, sequenceid=37, compaction requested=false 2024-11-27T12:26:18,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2603): Flush status journal for 3144c5c576970df3a2a1c91af06cc315: 2024-11-27T12:26:18,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315. 2024-11-27T12:26:18,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=124 2024-11-27T12:26:18,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.HMaster(4169): Remote procedure done, pid=124 2024-11-27T12:26:18,409 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=124, resume processing ppid=123 2024-11-27T12:26:18,409 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=124, ppid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0840 sec 2024-11-27T12:26:18,410 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees in 1.0950 sec 2024-11-27T12:26:18,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-27T12:26:18,458 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T12:26:18,458 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-27T12:26:18,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] procedure2.ProcedureExecutor(1139): Stored pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees 2024-11-27T12:26:18,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-27T12:26:18,460 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T12:26:18,460 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T12:26:18,461 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=126, ppid=125, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T12:26:18,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-27T12:26:18,613 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-27T12:26:18,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315. 2024-11-27T12:26:18,613 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2902): Flushing 3144c5c576970df3a2a1c91af06cc315 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-27T12:26:18,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/.tmp/A/3d4b9f70af3144298f9b40975de374bd is 50, key is test_row_0/A:col10/1732710378007/Put/seqid=0 2024-11-27T12:26:18,622 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742272_1448 (size=12001) 2024-11-27T12:26:18,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(8855): Flush requested on 3144c5c576970df3a2a1c91af06cc315 2024-11-27T12:26:18,643 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315. as already flushing 2024-11-27T12:26:18,673 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:18,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39696 deadline: 1732710438663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:18,673 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:18,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39698 deadline: 1732710438672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:18,674 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:18,674 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:18,674 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:26:18,674 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:18,674 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:18,674 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:18,674 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:26:18,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39710 deadline: 1732710438672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:18,674 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:18,674 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:18,674 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:26:18,685 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:18,685 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:18,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39712 deadline: 1732710438673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:18,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39674 deadline: 1732710438673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:18,685 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:18,685 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:18,685 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:18,685 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:18,685 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:26:18,685 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:26:18,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-27T12:26:18,780 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:18,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39696 deadline: 1732710438778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:18,780 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:18,780 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:18,781 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:26:18,780 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:18,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39698 deadline: 1732710438778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:18,781 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:18,781 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:18,781 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:26:18,781 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:18,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39710 deadline: 1732710438778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:18,781 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:18,781 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:18,781 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:26:18,790 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:18,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39712 deadline: 1732710438788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:18,791 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:18,791 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:18,791 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:18,791 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:26:18,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39674 deadline: 1732710438788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:18,791 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:18,791 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:18,791 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:26:18,990 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:18,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39696 deadline: 1732710438987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:18,990 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:18,990 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:18,990 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:18,991 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:26:18,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39698 deadline: 1732710438988, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:18,991 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:18,991 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:18,991 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:18,991 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:26:18,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39710 deadline: 1732710438988, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:18,991 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:18,991 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:18,991 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:26:19,005 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:19,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39712 deadline: 1732710438998, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:19,006 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:19,006 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:19,006 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:19,006 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:26:19,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39674 deadline: 1732710438998, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:19,006 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:19,006 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:19,006 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:26:19,023 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/.tmp/A/3d4b9f70af3144298f9b40975de374bd 2024-11-27T12:26:19,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/.tmp/B/932d9a74e91e4a69a896780b684453ec is 50, key is test_row_0/B:col10/1732710378007/Put/seqid=0 2024-11-27T12:26:19,033 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742273_1449 (size=12001) 2024-11-27T12:26:19,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-27T12:26:19,302 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:19,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39696 deadline: 1732710439297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:19,302 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:19,302 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:19,302 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:19,302 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:26:19,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39698 deadline: 1732710439298, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:19,303 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:19,303 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:19,303 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:26:19,303 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:19,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39710 deadline: 1732710439298, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:19,303 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:19,303 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:19,303 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:26:19,313 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:19,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39712 deadline: 1732710439308, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:19,313 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:19,313 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:19,313 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:26:19,324 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:19,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39674 deadline: 1732710439318, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:19,325 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:19,325 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:19,325 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:26:19,433 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/.tmp/B/932d9a74e91e4a69a896780b684453ec 2024-11-27T12:26:19,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/.tmp/C/cfff5ebfbf6c40c980f019f20ae46bdb is 50, key is test_row_0/C:col10/1732710378007/Put/seqid=0 2024-11-27T12:26:19,441 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742274_1450 (size=12001) 2024-11-27T12:26:19,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-27T12:26:19,812 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:19,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39696 deadline: 1732710439808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:19,812 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:19,812 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:19,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39698 deadline: 1732710439808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:19,812 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:19,812 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:26:19,813 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:19,813 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:19,813 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:19,813 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:26:19,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39710 deadline: 1732710439808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:19,813 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:19,813 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:19,813 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:26:19,825 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:19,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39712 deadline: 1732710439818, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:19,825 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:19,825 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:19,825 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:26:19,842 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/.tmp/C/cfff5ebfbf6c40c980f019f20ae46bdb 2024-11-27T12:26:19,843 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:19,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39674 deadline: 1732710439837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:19,844 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:19,844 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:19,844 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:26:19,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/.tmp/A/3d4b9f70af3144298f9b40975de374bd as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/A/3d4b9f70af3144298f9b40975de374bd 2024-11-27T12:26:19,848 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/A/3d4b9f70af3144298f9b40975de374bd, entries=150, sequenceid=49, filesize=11.7 K 2024-11-27T12:26:19,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/.tmp/B/932d9a74e91e4a69a896780b684453ec as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/B/932d9a74e91e4a69a896780b684453ec 2024-11-27T12:26:19,851 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/B/932d9a74e91e4a69a896780b684453ec, entries=150, sequenceid=49, filesize=11.7 K 2024-11-27T12:26:19,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/.tmp/C/cfff5ebfbf6c40c980f019f20ae46bdb as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/C/cfff5ebfbf6c40c980f019f20ae46bdb 2024-11-27T12:26:19,854 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/C/cfff5ebfbf6c40c980f019f20ae46bdb, entries=150, sequenceid=49, filesize=11.7 K 2024-11-27T12:26:19,855 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 3144c5c576970df3a2a1c91af06cc315 in 1242ms, sequenceid=49, compaction requested=true 2024-11-27T12:26:19,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2603): Flush status journal for 3144c5c576970df3a2a1c91af06cc315: 2024-11-27T12:26:19,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315. 2024-11-27T12:26:19,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=126 2024-11-27T12:26:19,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.HMaster(4169): Remote procedure done, pid=126 2024-11-27T12:26:19,857 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=126, resume processing ppid=125 2024-11-27T12:26:19,857 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=126, ppid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3950 sec 2024-11-27T12:26:19,858 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees in 1.3990 sec 2024-11-27T12:26:20,206 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-27T12:26:20,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-27T12:26:20,608 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T12:26:20,608 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-27T12:26:20,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] procedure2.ProcedureExecutor(1139): Stored pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees 2024-11-27T12:26:20,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-27T12:26:20,610 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T12:26:20,610 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T12:26:20,610 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=128, ppid=127, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T12:26:20,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-27T12:26:20,762 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-27T12:26:20,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315. 2024-11-27T12:26:20,762 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2902): Flushing 3144c5c576970df3a2a1c91af06cc315 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-27T12:26:20,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/.tmp/A/3a0f14592fc7447496497a5e0e21868b is 50, key is test_row_0/A:col10/1732710378672/Put/seqid=0 2024-11-27T12:26:20,770 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742275_1451 (size=12001) 2024-11-27T12:26:20,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(8855): Flush requested on 3144c5c576970df3a2a1c91af06cc315 2024-11-27T12:26:20,826 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315. as already flushing 2024-11-27T12:26:20,848 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:20,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39712 deadline: 1732710440838, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:20,849 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 1ba5af43fd99,36251,1732710241631 for 'test_row_1' in 3144c5c576970df3a2a1c91af06cc315 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2175 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T12:26:20,849 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:20,849 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:20,849 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:26:20,849 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:20,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39698 deadline: 1732710440842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:20,850 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:20,850 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:20,850 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:26:20,854 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:20,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39710 deadline: 1732710440847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:20,855 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:20,855 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:20,855 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:26:20,858 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:20,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39696 deadline: 1732710440848, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:20,858 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:20,858 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:20,858 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:26:20,866 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:20,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39674 deadline: 1732710440858, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:20,867 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 1ba5af43fd99,36251,1732710241631 for 'test_row_2' in 3144c5c576970df3a2a1c91af06cc315 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2193 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T12:26:20,867 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:20,867 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:20,867 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:26:20,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-27T12:26:20,958 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:20,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39698 deadline: 1732710440958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:20,959 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:20,959 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:20,959 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:26:20,965 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:20,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39710 deadline: 1732710440958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:20,965 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:20,965 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:20,965 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:26:20,975 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:20,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39696 deadline: 1732710440968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:20,975 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:20,975 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:20,975 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:26:21,170 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=73 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/.tmp/A/3a0f14592fc7447496497a5e0e21868b 2024-11-27T12:26:21,171 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:21,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39698 deadline: 1732710441168, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:21,172 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:21,172 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:21,171 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:21,172 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:26:21,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39710 deadline: 1732710441168, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:21,172 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:21,172 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:21,172 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:26:21,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/.tmp/B/516c4743ce1a4ea89d0d72b15f4206c8 is 50, key is test_row_0/B:col10/1732710378672/Put/seqid=0 2024-11-27T12:26:21,184 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742276_1452 (size=12001) 2024-11-27T12:26:21,192 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:21,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39696 deadline: 1732710441188, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:21,192 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:21,192 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:21,192 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:26:21,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-27T12:26:21,484 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:21,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39698 deadline: 1732710441477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:21,484 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:21,484 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:21,484 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:21,484 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:26:21,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39710 deadline: 1732710441478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:21,485 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:21,485 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:21,485 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:26:21,505 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:21,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39696 deadline: 1732710441498, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:21,506 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:21,506 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:21,506 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:26:21,585 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=73 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/.tmp/B/516c4743ce1a4ea89d0d72b15f4206c8 2024-11-27T12:26:21,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/.tmp/C/cbb1368c48e04278866da73ff790213a is 50, key is test_row_0/C:col10/1732710378672/Put/seqid=0 2024-11-27T12:26:21,594 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742277_1453 (size=12001) 2024-11-27T12:26:21,595 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=73 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/.tmp/C/cbb1368c48e04278866da73ff790213a 2024-11-27T12:26:21,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/.tmp/A/3a0f14592fc7447496497a5e0e21868b as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/A/3a0f14592fc7447496497a5e0e21868b 2024-11-27T12:26:21,602 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/A/3a0f14592fc7447496497a5e0e21868b, entries=150, sequenceid=73, filesize=11.7 K 2024-11-27T12:26:21,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/.tmp/B/516c4743ce1a4ea89d0d72b15f4206c8 as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/B/516c4743ce1a4ea89d0d72b15f4206c8 2024-11-27T12:26:21,605 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/B/516c4743ce1a4ea89d0d72b15f4206c8, entries=150, sequenceid=73, filesize=11.7 K 2024-11-27T12:26:21,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/.tmp/C/cbb1368c48e04278866da73ff790213a as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/C/cbb1368c48e04278866da73ff790213a 2024-11-27T12:26:21,608 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/C/cbb1368c48e04278866da73ff790213a, entries=150, sequenceid=73, filesize=11.7 K 2024-11-27T12:26:21,609 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 3144c5c576970df3a2a1c91af06cc315 in 847ms, sequenceid=73, compaction requested=true 2024-11-27T12:26:21,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2603): Flush status journal for 3144c5c576970df3a2a1c91af06cc315: 2024-11-27T12:26:21,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315. 2024-11-27T12:26:21,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=128 2024-11-27T12:26:21,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.HMaster(4169): Remote procedure done, pid=128 2024-11-27T12:26:21,611 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=128, resume processing ppid=127 2024-11-27T12:26:21,611 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=128, ppid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0000 sec 2024-11-27T12:26:21,612 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees in 1.0040 sec 2024-11-27T12:26:21,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-27T12:26:21,748 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T12:26:21,748 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-27T12:26:21,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] procedure2.ProcedureExecutor(1139): Stored pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees 2024-11-27T12:26:21,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-27T12:26:21,750 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T12:26:21,750 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T12:26:21,750 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=130, ppid=129, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T12:26:21,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-27T12:26:21,902 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-27T12:26:21,902 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315. 2024-11-27T12:26:21,903 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2902): Flushing 3144c5c576970df3a2a1c91af06cc315 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-27T12:26:21,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/.tmp/A/4426c21656524d38ac158d5c03ca56a8 is 50, key is test_row_0/A:col10/1732710380837/Put/seqid=0 2024-11-27T12:26:21,909 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742278_1454 (size=12001) 2024-11-27T12:26:21,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(8855): Flush requested on 3144c5c576970df3a2a1c91af06cc315 2024-11-27T12:26:21,993 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315. as already flushing 2024-11-27T12:26:22,058 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:22,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39698 deadline: 1732710442049, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:22,058 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:22,058 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:22,058 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:26:22,060 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:22,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39710 deadline: 1732710442051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:22,060 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:22,060 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:22,060 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:26:22,062 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:22,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39696 deadline: 1732710442058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:22,062 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:22,062 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:22,062 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:26:22,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-27T12:26:22,174 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:22,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39698 deadline: 1732710442167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:22,174 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:22,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39710 deadline: 1732710442168, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:22,174 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:22,174 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:22,174 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:26:22,174 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:22,174 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:22,174 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:22,174 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:26:22,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39696 deadline: 1732710442168, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:22,175 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:22,175 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:22,175 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:26:22,312 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=85 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/.tmp/A/4426c21656524d38ac158d5c03ca56a8 2024-11-27T12:26:22,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/.tmp/B/708eba06bb944b7c808ec309f8119081 is 50, key is test_row_0/B:col10/1732710380837/Put/seqid=0 2024-11-27T12:26:22,332 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742279_1455 (size=12001) 2024-11-27T12:26:22,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-27T12:26:22,386 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:22,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39698 deadline: 1732710442378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:22,386 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:22,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39710 deadline: 1732710442379, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:22,386 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:22,386 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:22,386 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:26:22,386 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:22,386 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:22,386 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:22,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39696 deadline: 1732710442379, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:22,386 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:26:22,387 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:22,387 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:22,387 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:26:22,696 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:22,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39698 deadline: 1732710442688, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:22,697 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:22,697 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:22,697 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:26:22,707 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:22,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39710 deadline: 1732710442698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:22,707 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:22,707 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:22,707 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:22,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39696 deadline: 1732710442698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:22,707 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:26:22,708 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:22,708 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:22,708 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:26:22,733 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=85 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/.tmp/B/708eba06bb944b7c808ec309f8119081 2024-11-27T12:26:22,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/.tmp/C/f61f7010e88441ef8a3b37f74e39c4f6 is 50, key is test_row_0/C:col10/1732710380837/Put/seqid=0 2024-11-27T12:26:22,821 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742280_1456 (size=12001) 2024-11-27T12:26:22,883 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:22,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39712 deadline: 1732710442878, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:22,883 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 1ba5af43fd99,36251,1732710241631 for 'test_row_1' in 3144c5c576970df3a2a1c91af06cc315 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4210 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T12:26:22,884 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:22,884 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:22,884 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:26:22,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-27T12:26:22,894 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:22,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39674 deadline: 1732710442888, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:22,894 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 1ba5af43fd99,36251,1732710241631 for 'test_row_2' in 3144c5c576970df3a2a1c91af06cc315 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4221 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T12:26:22,895 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:22,895 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:22,895 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:26:23,215 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:23,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39698 deadline: 1732710443208, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:23,216 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:23,216 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:23,216 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:26:23,222 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=85 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/.tmp/C/f61f7010e88441ef8a3b37f74e39c4f6 2024-11-27T12:26:23,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/.tmp/A/4426c21656524d38ac158d5c03ca56a8 as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/A/4426c21656524d38ac158d5c03ca56a8 2024-11-27T12:26:23,228 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/A/4426c21656524d38ac158d5c03ca56a8, entries=150, sequenceid=85, filesize=11.7 K 2024-11-27T12:26:23,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/.tmp/B/708eba06bb944b7c808ec309f8119081 as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/B/708eba06bb944b7c808ec309f8119081 2024-11-27T12:26:23,231 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/B/708eba06bb944b7c808ec309f8119081, entries=150, sequenceid=85, filesize=11.7 K 2024-11-27T12:26:23,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/.tmp/C/f61f7010e88441ef8a3b37f74e39c4f6 as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/C/f61f7010e88441ef8a3b37f74e39c4f6 2024-11-27T12:26:23,233 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:23,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39710 deadline: 1732710443217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:23,233 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:23,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39696 deadline: 1732710443217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:23,233 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:23,233 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:23,233 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:26:23,233 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:23,233 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:23,233 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:26:23,235 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/C/f61f7010e88441ef8a3b37f74e39c4f6, entries=150, sequenceid=85, filesize=11.7 K 2024-11-27T12:26:23,236 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 3144c5c576970df3a2a1c91af06cc315 in 1334ms, sequenceid=85, compaction requested=true 2024-11-27T12:26:23,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2603): Flush status journal for 3144c5c576970df3a2a1c91af06cc315: 2024-11-27T12:26:23,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315. 2024-11-27T12:26:23,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=130 2024-11-27T12:26:23,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.HMaster(4169): Remote procedure done, pid=130 2024-11-27T12:26:23,239 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=130, resume processing ppid=129 2024-11-27T12:26:23,239 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=130, ppid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4870 sec 2024-11-27T12:26:23,240 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees in 1.4910 sec 2024-11-27T12:26:23,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-27T12:26:23,898 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T12:26:23,899 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-27T12:26:23,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] procedure2.ProcedureExecutor(1139): Stored pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees 2024-11-27T12:26:23,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-27T12:26:23,900 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T12:26:23,901 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T12:26:23,901 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=132, ppid=131, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T12:26:24,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-27T12:26:24,052 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-27T12:26:24,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315. 2024-11-27T12:26:24,053 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2902): Flushing 3144c5c576970df3a2a1c91af06cc315 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-27T12:26:24,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/.tmp/A/78e69fc9eff646f2aa85eb5e7c6cf155 is 50, key is test_row_0/A:col10/1732710382057/Put/seqid=0 2024-11-27T12:26:24,064 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742281_1457 (size=12001) 2024-11-27T12:26:24,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-27T12:26:24,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(8855): Flush requested on 3144c5c576970df3a2a1c91af06cc315 2024-11-27T12:26:24,224 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315. as already flushing 2024-11-27T12:26:24,262 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:24,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39710 deadline: 1732710444256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:24,263 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:24,263 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:24,263 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:26:24,268 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:24,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39696 deadline: 1732710444257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:24,268 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:24,268 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:24,269 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:24,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39698 deadline: 1732710444260, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:24,269 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:26:24,269 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:24,269 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:24,269 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:26:24,377 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:24,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39710 deadline: 1732710444368, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:24,378 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:24,378 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:24,378 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:26:24,383 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:24,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39696 deadline: 1732710444378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:24,383 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:24,383 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:24,383 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:24,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39698 deadline: 1732710444378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:24,383 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:26:24,383 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:24,383 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:24,383 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:26:24,464 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=109 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/.tmp/A/78e69fc9eff646f2aa85eb5e7c6cf155 2024-11-27T12:26:24,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/.tmp/B/71dee8d1146f4c8195ab63e5d96e1441 is 50, key is test_row_0/B:col10/1732710382057/Put/seqid=0 2024-11-27T12:26:24,474 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742282_1458 (size=12001) 2024-11-27T12:26:24,474 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=109 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/.tmp/B/71dee8d1146f4c8195ab63e5d96e1441 2024-11-27T12:26:24,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/.tmp/C/6516a82bdf2548daa994dee46975d1be is 50, key is test_row_0/C:col10/1732710382057/Put/seqid=0 2024-11-27T12:26:24,482 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742283_1459 (size=12001) 2024-11-27T12:26:24,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-27T12:26:24,594 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:24,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39710 deadline: 1732710444588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:24,594 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:24,594 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:24,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39696 deadline: 1732710444588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:24,594 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:24,595 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:26:24,595 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:24,595 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:24,595 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:24,595 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:26:24,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39698 deadline: 1732710444588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:24,595 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:24,595 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:24,595 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:26:24,883 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=109 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/.tmp/C/6516a82bdf2548daa994dee46975d1be 2024-11-27T12:26:24,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/.tmp/A/78e69fc9eff646f2aa85eb5e7c6cf155 as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/A/78e69fc9eff646f2aa85eb5e7c6cf155 2024-11-27T12:26:24,889 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/A/78e69fc9eff646f2aa85eb5e7c6cf155, entries=150, sequenceid=109, filesize=11.7 K 2024-11-27T12:26:24,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/.tmp/B/71dee8d1146f4c8195ab63e5d96e1441 as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/B/71dee8d1146f4c8195ab63e5d96e1441 2024-11-27T12:26:24,892 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/B/71dee8d1146f4c8195ab63e5d96e1441, entries=150, sequenceid=109, filesize=11.7 K 2024-11-27T12:26:24,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/.tmp/C/6516a82bdf2548daa994dee46975d1be as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/C/6516a82bdf2548daa994dee46975d1be 2024-11-27T12:26:24,899 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/C/6516a82bdf2548daa994dee46975d1be, entries=150, sequenceid=109, filesize=11.7 K 2024-11-27T12:26:24,900 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 3144c5c576970df3a2a1c91af06cc315 in 846ms, sequenceid=109, compaction requested=true 2024-11-27T12:26:24,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2603): Flush status journal for 3144c5c576970df3a2a1c91af06cc315: 2024-11-27T12:26:24,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315. 2024-11-27T12:26:24,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=132 2024-11-27T12:26:24,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.HMaster(4169): Remote procedure done, pid=132 2024-11-27T12:26:24,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(8855): Flush requested on 3144c5c576970df3a2a1c91af06cc315 2024-11-27T12:26:24,902 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T12:26:24,902 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3144c5c576970df3a2a1c91af06cc315 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-27T12:26:24,903 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=132, resume processing ppid=131 2024-11-27T12:26:24,903 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=132, ppid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0000 sec 2024-11-27T12:26:24,905 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees in 1.0040 sec 2024-11-27T12:26:24,907 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/.tmp/A/b8dfae2a2a2e4ab88f18bdd1736c3fa9 is 50, key is test_row_0/A:col10/1732710384259/Put/seqid=0 2024-11-27T12:26:24,913 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742284_1460 (size=14341) 2024-11-27T12:26:24,914 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=122 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/.tmp/A/b8dfae2a2a2e4ab88f18bdd1736c3fa9 2024-11-27T12:26:24,921 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/.tmp/B/fc1ee9fac4964203a81d7ee7c9b76985 is 50, key is test_row_0/B:col10/1732710384259/Put/seqid=0 2024-11-27T12:26:24,924 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742285_1461 (size=12001) 2024-11-27T12:26:24,926 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=122 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/.tmp/B/fc1ee9fac4964203a81d7ee7c9b76985 2024-11-27T12:26:24,932 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/.tmp/C/6091c17e2d2e4927902eed2303c2887d is 50, key is test_row_0/C:col10/1732710384259/Put/seqid=0 2024-11-27T12:26:24,937 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742286_1462 (size=12001) 2024-11-27T12:26:24,995 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:24,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39698 deadline: 1732710444983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:24,996 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:24,996 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:24,996 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:26:24,996 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:24,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39710 deadline: 1732710444984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:24,996 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:24,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39696 deadline: 1732710444992, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:24,997 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:24,997 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:24,997 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:26:24,997 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:24,997 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:24,997 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:26:25,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-27T12:26:25,038 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T12:26:25,039 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-27T12:26:25,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] procedure2.ProcedureExecutor(1139): Stored pid=133, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees 2024-11-27T12:26:25,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-27T12:26:25,040 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=133, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T12:26:25,041 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=133, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T12:26:25,041 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=134, ppid=133, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T12:26:25,108 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:25,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39698 deadline: 1732710445098, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:25,109 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:25,109 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:25,109 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:26:25,118 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:25,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39710 deadline: 1732710445108, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:25,118 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:25,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39696 deadline: 1732710445108, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:25,118 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:25,118 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:25,118 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:26:25,118 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:25,118 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:25,118 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:26:25,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-27T12:26:25,193 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-27T12:26:25,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315. 2024-11-27T12:26:25,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315. as already flushing 2024-11-27T12:26:25,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315. 2024-11-27T12:26:25,193 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 3144c5c576970df3a2a1c91af06cc315, NAME => 'TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:26:25,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 3144c5c576970df3a2a1c91af06cc315, NAME => 'TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:26:25,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3144c5c576970df3a2a1c91af06cc315, NAME => 'TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3144c5c576970df3a2a1c91af06cc315, NAME => 'TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:26:25,324 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:25,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39698 deadline: 1732710445318, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:25,325 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:25,325 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:25,325 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:26:25,337 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:25,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39710 deadline: 1732710445328, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:25,338 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=122 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/.tmp/C/6091c17e2d2e4927902eed2303c2887d 2024-11-27T12:26:25,338 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:25,338 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:25,338 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:25,338 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:26:25,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39696 deadline: 1732710445328, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:25,338 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:25,338 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:25,338 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:26:25,342 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/.tmp/A/b8dfae2a2a2e4ab88f18bdd1736c3fa9 as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/A/b8dfae2a2a2e4ab88f18bdd1736c3fa9 2024-11-27T12:26:25,345 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/A/b8dfae2a2a2e4ab88f18bdd1736c3fa9, entries=200, sequenceid=122, filesize=14.0 K 2024-11-27T12:26:25,345 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/.tmp/B/fc1ee9fac4964203a81d7ee7c9b76985 as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/B/fc1ee9fac4964203a81d7ee7c9b76985 2024-11-27T12:26:25,346 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-27T12:26:25,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315. 2024-11-27T12:26:25,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315. as already flushing 2024-11-27T12:26:25,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315. 2024-11-27T12:26:25,346 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 3144c5c576970df3a2a1c91af06cc315, NAME => 'TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:26:25,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 3144c5c576970df3a2a1c91af06cc315, NAME => 'TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:26:25,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3144c5c576970df3a2a1c91af06cc315, NAME => 'TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3144c5c576970df3a2a1c91af06cc315, NAME => 'TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:26:25,350 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/B/fc1ee9fac4964203a81d7ee7c9b76985, entries=150, sequenceid=122, filesize=11.7 K 2024-11-27T12:26:25,351 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/.tmp/C/6091c17e2d2e4927902eed2303c2887d as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/C/6091c17e2d2e4927902eed2303c2887d 2024-11-27T12:26:25,354 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/C/6091c17e2d2e4927902eed2303c2887d, entries=150, sequenceid=122, filesize=11.7 K 2024-11-27T12:26:25,355 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 3144c5c576970df3a2a1c91af06cc315 in 453ms, sequenceid=122, compaction requested=true 2024-11-27T12:26:25,355 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3144c5c576970df3a2a1c91af06cc315: 2024-11-27T12:26:25,355 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 7 store files, 0 compacting, 7 eligible, 16 blocking 2024-11-27T12:26:25,356 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3144c5c576970df3a2a1c91af06cc315:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T12:26:25,356 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T12:26:25,356 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3144c5c576970df3a2a1c91af06cc315:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T12:26:25,356 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T12:26:25,356 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 7 store files, 0 compacting, 7 eligible, 16 blocking 2024-11-27T12:26:25,356 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3144c5c576970df3a2a1c91af06cc315:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T12:26:25,356 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T12:26:25,357 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 7 files of size 91027 starting at candidate #0 after considering 15 permutations with 15 in ratio 2024-11-27T12:26:25,357 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HStore(1541): 3144c5c576970df3a2a1c91af06cc315/A is initiating minor compaction (all files) 2024-11-27T12:26:25,357 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3144c5c576970df3a2a1c91af06cc315/A in TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315. 2024-11-27T12:26:25,357 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 7 files of size 84007 starting at candidate #0 after considering 15 permutations with 15 in ratio 2024-11-27T12:26:25,357 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HStore(1541): 3144c5c576970df3a2a1c91af06cc315/B is initiating minor compaction (all files) 2024-11-27T12:26:25,357 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/A/b1a17dce207041728e6a4cfa67964ffa, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/A/8663aa07aae84174a946123426cd00bf, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/A/3d4b9f70af3144298f9b40975de374bd, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/A/3a0f14592fc7447496497a5e0e21868b, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/A/4426c21656524d38ac158d5c03ca56a8, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/A/78e69fc9eff646f2aa85eb5e7c6cf155, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/A/b8dfae2a2a2e4ab88f18bdd1736c3fa9] into tmpdir=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/.tmp, totalSize=88.9 K 2024-11-27T12:26:25,357 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3144c5c576970df3a2a1c91af06cc315/B in TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315. 2024-11-27T12:26:25,357 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/B/eb948be447f54b5eb6b2b773996a6e84, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/B/1eaa295f51194d919d5c4a218f596055, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/B/932d9a74e91e4a69a896780b684453ec, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/B/516c4743ce1a4ea89d0d72b15f4206c8, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/B/708eba06bb944b7c808ec309f8119081, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/B/71dee8d1146f4c8195ab63e5d96e1441, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/B/fc1ee9fac4964203a81d7ee7c9b76985] into tmpdir=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/.tmp, totalSize=82.0 K 2024-11-27T12:26:25,357 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] compactions.Compactor(225): Compacting b1a17dce207041728e6a4cfa67964ffa, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1732710377321 2024-11-27T12:26:25,357 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.Compactor(225): Compacting eb948be447f54b5eb6b2b773996a6e84, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1732710377324 2024-11-27T12:26:25,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-27T12:26:25,358 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8663aa07aae84174a946123426cd00bf, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1732710377359 2024-11-27T12:26:25,358 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.Compactor(225): Compacting 1eaa295f51194d919d5c4a218f596055, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1732710377359 2024-11-27T12:26:25,358 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3d4b9f70af3144298f9b40975de374bd, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1732710377998 2024-11-27T12:26:25,358 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.Compactor(225): Compacting 932d9a74e91e4a69a896780b684453ec, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1732710377998 2024-11-27T12:26:25,358 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3a0f14592fc7447496497a5e0e21868b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=73, earliestPutTs=1732710378662 2024-11-27T12:26:25,358 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.Compactor(225): Compacting 516c4743ce1a4ea89d0d72b15f4206c8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=73, earliestPutTs=1732710378662 2024-11-27T12:26:25,358 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4426c21656524d38ac158d5c03ca56a8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=85, earliestPutTs=1732710380837 2024-11-27T12:26:25,358 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.Compactor(225): Compacting 708eba06bb944b7c808ec309f8119081, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=85, earliestPutTs=1732710380837 2024-11-27T12:26:25,358 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] compactions.Compactor(225): Compacting 78e69fc9eff646f2aa85eb5e7c6cf155, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=109, earliestPutTs=1732710382044 2024-11-27T12:26:25,359 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.Compactor(225): Compacting 71dee8d1146f4c8195ab63e5d96e1441, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=109, earliestPutTs=1732710382044 2024-11-27T12:26:25,359 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] compactions.Compactor(225): Compacting b8dfae2a2a2e4ab88f18bdd1736c3fa9, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=122, earliestPutTs=1732710384255 2024-11-27T12:26:25,359 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.Compactor(225): Compacting fc1ee9fac4964203a81d7ee7c9b76985, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=122, earliestPutTs=1732710384256 2024-11-27T12:26:25,373 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3144c5c576970df3a2a1c91af06cc315#B#compaction#383 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 53.85 MB/second 2024-11-27T12:26:25,373 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/.tmp/B/b7c7b8c43e434a99a7c18e3ca4543307 is 50, key is test_row_0/B:col10/1732710384259/Put/seqid=0 2024-11-27T12:26:25,376 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3144c5c576970df3a2a1c91af06cc315#A#compaction#384 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-27T12:26:25,377 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/.tmp/A/1a8531bdd0574cc2af75b4fa9e98b2b1 is 50, key is test_row_0/A:col10/1732710384259/Put/seqid=0 2024-11-27T12:26:25,377 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742287_1463 (size=12241) 2024-11-27T12:26:25,380 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742288_1464 (size=12241) 2024-11-27T12:26:25,384 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/.tmp/A/1a8531bdd0574cc2af75b4fa9e98b2b1 as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/A/1a8531bdd0574cc2af75b4fa9e98b2b1 2024-11-27T12:26:25,387 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 7 (all) file(s) in 3144c5c576970df3a2a1c91af06cc315/A of 3144c5c576970df3a2a1c91af06cc315 into 1a8531bdd0574cc2af75b4fa9e98b2b1(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T12:26:25,387 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3144c5c576970df3a2a1c91af06cc315: 2024-11-27T12:26:25,388 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., storeName=3144c5c576970df3a2a1c91af06cc315/A, priority=9, startTime=1732710385355; duration=0sec 2024-11-27T12:26:25,388 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T12:26:25,388 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3144c5c576970df3a2a1c91af06cc315:A 2024-11-27T12:26:25,388 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 7 store files, 0 compacting, 7 eligible, 16 blocking 2024-11-27T12:26:25,389 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 7 files of size 84007 starting at candidate #0 after considering 15 permutations with 15 in ratio 2024-11-27T12:26:25,389 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HStore(1541): 3144c5c576970df3a2a1c91af06cc315/C is initiating minor compaction (all files) 2024-11-27T12:26:25,389 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3144c5c576970df3a2a1c91af06cc315/C in TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315. 2024-11-27T12:26:25,389 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/C/16b0a48d92ae45c5900bdde6eaecfdb2, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/C/ae5edde338334cd4a72f5a131da0389c, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/C/cfff5ebfbf6c40c980f019f20ae46bdb, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/C/cbb1368c48e04278866da73ff790213a, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/C/f61f7010e88441ef8a3b37f74e39c4f6, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/C/6516a82bdf2548daa994dee46975d1be, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/C/6091c17e2d2e4927902eed2303c2887d] into tmpdir=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/.tmp, totalSize=82.0 K 2024-11-27T12:26:25,389 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] compactions.Compactor(225): Compacting 16b0a48d92ae45c5900bdde6eaecfdb2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1732710377324 2024-11-27T12:26:25,389 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] compactions.Compactor(225): Compacting ae5edde338334cd4a72f5a131da0389c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1732710377359 2024-11-27T12:26:25,390 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] compactions.Compactor(225): Compacting cfff5ebfbf6c40c980f019f20ae46bdb, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1732710377998 2024-11-27T12:26:25,390 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] compactions.Compactor(225): Compacting cbb1368c48e04278866da73ff790213a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=73, earliestPutTs=1732710378662 2024-11-27T12:26:25,390 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] compactions.Compactor(225): Compacting f61f7010e88441ef8a3b37f74e39c4f6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=85, earliestPutTs=1732710380837 2024-11-27T12:26:25,390 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6516a82bdf2548daa994dee46975d1be, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=109, earliestPutTs=1732710382044 2024-11-27T12:26:25,391 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6091c17e2d2e4927902eed2303c2887d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=122, earliestPutTs=1732710384256 2024-11-27T12:26:25,402 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3144c5c576970df3a2a1c91af06cc315#C#compaction#385 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-27T12:26:25,403 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/.tmp/C/6399fa4932bb4b15a6a2c61f9556efb3 is 50, key is test_row_0/C:col10/1732710384259/Put/seqid=0 2024-11-27T12:26:25,409 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742289_1465 (size=12241) 2024-11-27T12:26:25,416 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/.tmp/C/6399fa4932bb4b15a6a2c61f9556efb3 as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/C/6399fa4932bb4b15a6a2c61f9556efb3 2024-11-27T12:26:25,422 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 7 (all) file(s) in 3144c5c576970df3a2a1c91af06cc315/C of 3144c5c576970df3a2a1c91af06cc315 into 6399fa4932bb4b15a6a2c61f9556efb3(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T12:26:25,422 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3144c5c576970df3a2a1c91af06cc315: 2024-11-27T12:26:25,422 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., storeName=3144c5c576970df3a2a1c91af06cc315/C, priority=9, startTime=1732710385356; duration=0sec 2024-11-27T12:26:25,422 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T12:26:25,422 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3144c5c576970df3a2a1c91af06cc315:C 2024-11-27T12:26:25,501 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-27T12:26:25,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315. 2024-11-27T12:26:25,501 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2902): Flushing 3144c5c576970df3a2a1c91af06cc315 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-27T12:26:25,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/.tmp/A/bad67b95594243199cb28fb002eb977e is 50, key is test_row_0/A:col10/1732710384983/Put/seqid=0 2024-11-27T12:26:25,517 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742290_1466 (size=12151) 2024-11-27T12:26:25,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(8855): Flush requested on 3144c5c576970df3a2a1c91af06cc315 2024-11-27T12:26:25,632 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315. as already flushing 2024-11-27T12:26:25,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-27T12:26:25,672 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:25,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39710 deadline: 1732710445664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:25,672 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:25,672 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:25,672 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:26:25,674 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:25,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39696 deadline: 1732710445672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:25,674 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:25,675 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:25,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39698 deadline: 1732710445672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:25,675 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:25,675 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:26:25,675 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:25,675 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:25,675 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:26:25,781 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/.tmp/B/b7c7b8c43e434a99a7c18e3ca4543307 as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/B/b7c7b8c43e434a99a7c18e3ca4543307 2024-11-27T12:26:25,784 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 7 (all) file(s) in 3144c5c576970df3a2a1c91af06cc315/B of 3144c5c576970df3a2a1c91af06cc315 into b7c7b8c43e434a99a7c18e3ca4543307(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T12:26:25,784 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3144c5c576970df3a2a1c91af06cc315: 2024-11-27T12:26:25,784 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., storeName=3144c5c576970df3a2a1c91af06cc315/B, priority=9, startTime=1732710385356; duration=0sec 2024-11-27T12:26:25,784 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T12:26:25,784 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3144c5c576970df3a2a1c91af06cc315:B 2024-11-27T12:26:25,787 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:25,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39710 deadline: 1732710445778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:25,788 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:25,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39696 deadline: 1732710445778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:25,788 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:25,788 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:25,788 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:26:25,788 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:25,788 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:25,788 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:26:25,788 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:25,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39698 deadline: 1732710445778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:25,789 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:25,789 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:25,789 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:26:25,918 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=147 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/.tmp/A/bad67b95594243199cb28fb002eb977e 2024-11-27T12:26:25,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/.tmp/B/4c7aa0b877704768b07e46e9779b736c is 50, key is test_row_0/B:col10/1732710384983/Put/seqid=0 2024-11-27T12:26:25,928 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742291_1467 (size=12151) 2024-11-27T12:26:26,002 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:26,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39710 deadline: 1732710445998, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:26,002 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:26,002 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:26,002 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:26,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39696 deadline: 1732710445998, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:26,002 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:26:26,002 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:26,003 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:26,003 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:26,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39698 deadline: 1732710445998, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:26,003 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:26:26,003 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:26,003 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:26,003 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:26:26,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-27T12:26:26,313 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:26,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39710 deadline: 1732710446307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:26,313 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:26,313 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:26,313 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:26,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39696 deadline: 1732710446308, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:26,313 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:26:26,314 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:26,314 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:26,314 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:26,314 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:26:26,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39698 deadline: 1732710446308, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:26,314 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:26,314 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:26,314 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:26:26,328 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=147 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/.tmp/B/4c7aa0b877704768b07e46e9779b736c 2024-11-27T12:26:26,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/.tmp/C/0007d7093f65428e8372f78cd84daf9b is 50, key is test_row_0/C:col10/1732710384983/Put/seqid=0 2024-11-27T12:26:26,337 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742292_1468 (size=12151) 2024-11-27T12:26:26,738 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=147 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/.tmp/C/0007d7093f65428e8372f78cd84daf9b 2024-11-27T12:26:26,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/.tmp/A/bad67b95594243199cb28fb002eb977e as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/A/bad67b95594243199cb28fb002eb977e 2024-11-27T12:26:26,744 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/A/bad67b95594243199cb28fb002eb977e, entries=150, sequenceid=147, filesize=11.9 K 2024-11-27T12:26:26,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/.tmp/B/4c7aa0b877704768b07e46e9779b736c as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/B/4c7aa0b877704768b07e46e9779b736c 2024-11-27T12:26:26,747 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/B/4c7aa0b877704768b07e46e9779b736c, entries=150, sequenceid=147, filesize=11.9 K 2024-11-27T12:26:26,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/.tmp/C/0007d7093f65428e8372f78cd84daf9b as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/C/0007d7093f65428e8372f78cd84daf9b 2024-11-27T12:26:26,751 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/C/0007d7093f65428e8372f78cd84daf9b, entries=150, sequenceid=147, filesize=11.9 K 2024-11-27T12:26:26,752 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=73.80 KB/75570 for 3144c5c576970df3a2a1c91af06cc315 in 1250ms, sequenceid=147, compaction requested=false 2024-11-27T12:26:26,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2603): Flush status journal for 3144c5c576970df3a2a1c91af06cc315: 2024-11-27T12:26:26,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315. 2024-11-27T12:26:26,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=134 2024-11-27T12:26:26,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.HMaster(4169): Remote procedure done, pid=134 2024-11-27T12:26:26,756 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=134, resume processing ppid=133 2024-11-27T12:26:26,756 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=134, ppid=133, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7140 sec 2024-11-27T12:26:26,758 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=133, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees in 1.7180 sec 2024-11-27T12:26:26,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(8855): Flush requested on 3144c5c576970df3a2a1c91af06cc315 2024-11-27T12:26:26,822 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T12:26:26,822 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3144c5c576970df3a2a1c91af06cc315 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-27T12:26:26,827 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/.tmp/A/d9167162e3a0497bb0ecc171e7887022 is 50, key is test_row_0/A:col10/1732710386822/Put/seqid=0 2024-11-27T12:26:26,830 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742293_1469 (size=12151) 2024-11-27T12:26:26,861 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:26,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39696 deadline: 1732710446857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:26,861 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:26,861 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:26,861 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:26,861 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:26:26,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39698 deadline: 1732710446858, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:26,862 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:26,862 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:26,862 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:26:26,867 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:26,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39710 deadline: 1732710446858, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:26,867 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:26,867 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:26,867 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:26:26,895 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:26,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39712 deadline: 1732710446888, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:26,895 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 1ba5af43fd99,36251,1732710241631 for 'test_row_1' in 3144c5c576970df3a2a1c91af06cc315 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8222 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T12:26:26,896 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:26,896 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:26,896 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:26:26,935 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:26,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39674 deadline: 1732710446928, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:26,936 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 1ba5af43fd99,36251,1732710241631 for 'test_row_2' in 3144c5c576970df3a2a1c91af06cc315 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8262 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T12:26:26,936 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:26,936 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:26,936 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:26:26,974 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:26,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39696 deadline: 1732710446968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:26,974 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:26,974 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:26,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39698 deadline: 1732710446968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:26,974 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:26,974 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:26:26,975 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:26,975 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:26,975 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:26:26,984 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:26,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39710 deadline: 1732710446977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:26,985 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:26,985 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:26,985 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:26:27,183 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:27,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39696 deadline: 1732710447178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:27,184 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:27,184 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:27,184 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:27,184 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:26:27,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39698 deadline: 1732710447178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:27,184 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:27,184 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:27,184 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:26:27,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-27T12:26:27,188 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T12:26:27,189 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-27T12:26:27,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] procedure2.ProcedureExecutor(1139): Stored pid=135, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees 2024-11-27T12:26:27,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-27T12:26:27,190 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=135, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T12:26:27,191 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=135, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T12:26:27,191 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=136, ppid=135, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T12:26:27,195 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:27,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39710 deadline: 1732710447188, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:27,196 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:27,196 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:27,196 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:26:27,230 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=164 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/.tmp/A/d9167162e3a0497bb0ecc171e7887022 2024-11-27T12:26:27,237 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/.tmp/B/7fc7d60dd36a4deab80a8d4e046ac1e5 is 50, key is test_row_0/B:col10/1732710386822/Put/seqid=0 2024-11-27T12:26:27,240 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742294_1470 (size=12151) 2024-11-27T12:26:27,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-27T12:26:27,343 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-27T12:26:27,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315. 2024-11-27T12:26:27,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315. as already flushing 2024-11-27T12:26:27,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315. 2024-11-27T12:26:27,343 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 3144c5c576970df3a2a1c91af06cc315, NAME => 'TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:26:27,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 3144c5c576970df3a2a1c91af06cc315, NAME => 'TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:26:27,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3144c5c576970df3a2a1c91af06cc315, NAME => 'TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3144c5c576970df3a2a1c91af06cc315, NAME => 'TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:26:27,496 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-27T12:26:27,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315. 2024-11-27T12:26:27,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315. as already flushing 2024-11-27T12:26:27,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315. 2024-11-27T12:26:27,496 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 3144c5c576970df3a2a1c91af06cc315, NAME => 'TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:26:27,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 3144c5c576970df3a2a1c91af06cc315, NAME => 'TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:26:27,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3144c5c576970df3a2a1c91af06cc315, NAME => 'TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3144c5c576970df3a2a1c91af06cc315, NAME => 'TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:26:27,498 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:27,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39696 deadline: 1732710447488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:27,498 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:27,498 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:27,498 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:27,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39698 deadline: 1732710447488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:27,498 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:26:27,499 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:27,499 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:27,499 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:26:27,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-27T12:26:27,512 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:27,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39710 deadline: 1732710447508, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:27,512 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:27,512 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:27,512 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:26:27,641 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=164 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/.tmp/B/7fc7d60dd36a4deab80a8d4e046ac1e5 2024-11-27T12:26:27,647 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/.tmp/C/de863092a9e544d1bd0a430d366a1772 is 50, key is test_row_0/C:col10/1732710386822/Put/seqid=0 2024-11-27T12:26:27,648 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-27T12:26:27,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315. 2024-11-27T12:26:27,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315. as already flushing 2024-11-27T12:26:27,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315. 2024-11-27T12:26:27,649 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 3144c5c576970df3a2a1c91af06cc315, NAME => 'TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:26:27,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 3144c5c576970df3a2a1c91af06cc315, NAME => 'TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:26:27,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3144c5c576970df3a2a1c91af06cc315, NAME => 'TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3144c5c576970df3a2a1c91af06cc315, NAME => 'TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:26:27,650 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742295_1471 (size=12151) 2024-11-27T12:26:27,651 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=164 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/.tmp/C/de863092a9e544d1bd0a430d366a1772 2024-11-27T12:26:27,654 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/.tmp/A/d9167162e3a0497bb0ecc171e7887022 as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/A/d9167162e3a0497bb0ecc171e7887022 2024-11-27T12:26:27,657 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/A/d9167162e3a0497bb0ecc171e7887022, entries=150, sequenceid=164, filesize=11.9 K 2024-11-27T12:26:27,657 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/.tmp/B/7fc7d60dd36a4deab80a8d4e046ac1e5 as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/B/7fc7d60dd36a4deab80a8d4e046ac1e5 2024-11-27T12:26:27,660 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/B/7fc7d60dd36a4deab80a8d4e046ac1e5, entries=150, sequenceid=164, filesize=11.9 K 2024-11-27T12:26:27,660 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/.tmp/C/de863092a9e544d1bd0a430d366a1772 as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/C/de863092a9e544d1bd0a430d366a1772 2024-11-27T12:26:27,663 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/C/de863092a9e544d1bd0a430d366a1772, entries=150, sequenceid=164, filesize=11.9 K 2024-11-27T12:26:27,664 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=114.05 KB/116790 for 3144c5c576970df3a2a1c91af06cc315 in 842ms, sequenceid=164, compaction requested=true 2024-11-27T12:26:27,664 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3144c5c576970df3a2a1c91af06cc315: 2024-11-27T12:26:27,664 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3144c5c576970df3a2a1c91af06cc315:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T12:26:27,664 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T12:26:27,664 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3144c5c576970df3a2a1c91af06cc315:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T12:26:27,664 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T12:26:27,664 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T12:26:27,664 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T12:26:27,664 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3144c5c576970df3a2a1c91af06cc315:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T12:26:27,665 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T12:26:27,665 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36543 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T12:26:27,665 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36543 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T12:26:27,665 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HStore(1541): 3144c5c576970df3a2a1c91af06cc315/B is initiating minor compaction (all files) 2024-11-27T12:26:27,665 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HStore(1541): 3144c5c576970df3a2a1c91af06cc315/A is initiating minor compaction (all files) 2024-11-27T12:26:27,665 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3144c5c576970df3a2a1c91af06cc315/B in TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315. 2024-11-27T12:26:27,665 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3144c5c576970df3a2a1c91af06cc315/A in TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315. 2024-11-27T12:26:27,665 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/B/b7c7b8c43e434a99a7c18e3ca4543307, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/B/4c7aa0b877704768b07e46e9779b736c, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/B/7fc7d60dd36a4deab80a8d4e046ac1e5] into tmpdir=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/.tmp, totalSize=35.7 K 2024-11-27T12:26:27,665 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/A/1a8531bdd0574cc2af75b4fa9e98b2b1, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/A/bad67b95594243199cb28fb002eb977e, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/A/d9167162e3a0497bb0ecc171e7887022] into tmpdir=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/.tmp, totalSize=35.7 K 2024-11-27T12:26:27,666 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.Compactor(225): Compacting b7c7b8c43e434a99a7c18e3ca4543307, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=122, earliestPutTs=1732710384256 2024-11-27T12:26:27,666 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1a8531bdd0574cc2af75b4fa9e98b2b1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=122, earliestPutTs=1732710384256 2024-11-27T12:26:27,666 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] compactions.Compactor(225): Compacting bad67b95594243199cb28fb002eb977e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=147, earliestPutTs=1732710384982 2024-11-27T12:26:27,666 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.Compactor(225): Compacting 4c7aa0b877704768b07e46e9779b736c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=147, earliestPutTs=1732710384982 2024-11-27T12:26:27,666 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.Compactor(225): Compacting 7fc7d60dd36a4deab80a8d4e046ac1e5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=164, earliestPutTs=1732710385656 2024-11-27T12:26:27,666 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] compactions.Compactor(225): Compacting d9167162e3a0497bb0ecc171e7887022, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=164, earliestPutTs=1732710385656 2024-11-27T12:26:27,671 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3144c5c576970df3a2a1c91af06cc315#B#compaction#392 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 53.85 MB/second 2024-11-27T12:26:27,672 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3144c5c576970df3a2a1c91af06cc315#A#compaction#393 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-27T12:26:27,672 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/.tmp/B/d862d616b87640f386465b3289452b22 is 50, key is test_row_0/B:col10/1732710386822/Put/seqid=0 2024-11-27T12:26:27,672 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/.tmp/A/98c249cc1bde47f1b096c9a662048116 is 50, key is test_row_0/A:col10/1732710386822/Put/seqid=0 2024-11-27T12:26:27,675 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742296_1472 (size=12493) 2024-11-27T12:26:27,676 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742297_1473 (size=12493) 2024-11-27T12:26:27,684 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/.tmp/B/d862d616b87640f386465b3289452b22 as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/B/d862d616b87640f386465b3289452b22 2024-11-27T12:26:27,688 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3144c5c576970df3a2a1c91af06cc315/B of 3144c5c576970df3a2a1c91af06cc315 into d862d616b87640f386465b3289452b22(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T12:26:27,688 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3144c5c576970df3a2a1c91af06cc315: 2024-11-27T12:26:27,688 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., storeName=3144c5c576970df3a2a1c91af06cc315/B, priority=13, startTime=1732710387664; duration=0sec 2024-11-27T12:26:27,688 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T12:26:27,688 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3144c5c576970df3a2a1c91af06cc315:B 2024-11-27T12:26:27,688 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T12:26:27,689 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36543 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T12:26:27,689 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HStore(1541): 3144c5c576970df3a2a1c91af06cc315/C is initiating minor compaction (all files) 2024-11-27T12:26:27,689 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3144c5c576970df3a2a1c91af06cc315/C in TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315. 2024-11-27T12:26:27,689 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/C/6399fa4932bb4b15a6a2c61f9556efb3, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/C/0007d7093f65428e8372f78cd84daf9b, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/C/de863092a9e544d1bd0a430d366a1772] into tmpdir=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/.tmp, totalSize=35.7 K 2024-11-27T12:26:27,689 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.Compactor(225): Compacting 6399fa4932bb4b15a6a2c61f9556efb3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=122, earliestPutTs=1732710384256 2024-11-27T12:26:27,690 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.Compactor(225): Compacting 0007d7093f65428e8372f78cd84daf9b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=147, earliestPutTs=1732710384982 2024-11-27T12:26:27,690 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.Compactor(225): Compacting de863092a9e544d1bd0a430d366a1772, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=164, earliestPutTs=1732710385656 2024-11-27T12:26:27,694 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3144c5c576970df3a2a1c91af06cc315#C#compaction#394 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-27T12:26:27,695 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/.tmp/C/7dd9abb37bea4c45b38854bae479c2ae is 50, key is test_row_0/C:col10/1732710386822/Put/seqid=0 2024-11-27T12:26:27,699 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742298_1474 (size=12493) 2024-11-27T12:26:27,801 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-27T12:26:27,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315. 2024-11-27T12:26:27,801 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2902): Flushing 3144c5c576970df3a2a1c91af06cc315 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-27T12:26:27,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-27T12:26:27,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/.tmp/A/8bb3eacd4e7942e49e9628f358b7bdb0 is 50, key is test_row_0/A:col10/1732710386857/Put/seqid=0 2024-11-27T12:26:27,872 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742299_1475 (size=12151) 2024-11-27T12:26:28,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(8855): Flush requested on 3144c5c576970df3a2a1c91af06cc315 2024-11-27T12:26:28,010 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315. as already flushing 2024-11-27T12:26:28,037 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:28,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39696 deadline: 1732710448034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:28,037 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:28,037 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:28,037 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:26:28,043 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:28,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39710 deadline: 1732710448035, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:28,044 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:28,044 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:28,044 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:28,044 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:26:28,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39698 deadline: 1732710448037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:28,044 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:28,044 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:28,044 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:26:28,079 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/.tmp/A/98c249cc1bde47f1b096c9a662048116 as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/A/98c249cc1bde47f1b096c9a662048116 2024-11-27T12:26:28,083 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3144c5c576970df3a2a1c91af06cc315/A of 3144c5c576970df3a2a1c91af06cc315 into 98c249cc1bde47f1b096c9a662048116(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T12:26:28,083 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3144c5c576970df3a2a1c91af06cc315: 2024-11-27T12:26:28,083 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., storeName=3144c5c576970df3a2a1c91af06cc315/A, priority=13, startTime=1732710387664; duration=0sec 2024-11-27T12:26:28,083 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T12:26:28,083 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3144c5c576970df3a2a1c91af06cc315:A 2024-11-27T12:26:28,103 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/.tmp/C/7dd9abb37bea4c45b38854bae479c2ae as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/C/7dd9abb37bea4c45b38854bae479c2ae 2024-11-27T12:26:28,107 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3144c5c576970df3a2a1c91af06cc315/C of 3144c5c576970df3a2a1c91af06cc315 into 7dd9abb37bea4c45b38854bae479c2ae(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T12:26:28,107 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3144c5c576970df3a2a1c91af06cc315: 2024-11-27T12:26:28,107 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., storeName=3144c5c576970df3a2a1c91af06cc315/C, priority=13, startTime=1732710387664; duration=0sec 2024-11-27T12:26:28,107 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T12:26:28,107 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3144c5c576970df3a2a1c91af06cc315:C 2024-11-27T12:26:28,156 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:28,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39696 deadline: 1732710448148, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:28,157 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:28,157 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:28,157 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:28,157 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:26:28,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39710 deadline: 1732710448148, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:28,157 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:28,157 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:28,157 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:28,157 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:26:28,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39698 deadline: 1732710448148, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:28,158 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:28,158 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:28,158 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:26:28,273 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=185 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/.tmp/A/8bb3eacd4e7942e49e9628f358b7bdb0 2024-11-27T12:26:28,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/.tmp/B/caac659266c2481483bed5cf6a3a6ff3 is 50, key is test_row_0/B:col10/1732710386857/Put/seqid=0 2024-11-27T12:26:28,284 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742300_1476 (size=12151) 2024-11-27T12:26:28,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-27T12:26:28,372 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:28,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39696 deadline: 1732710448368, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:28,372 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:28,372 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:28,372 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:28,372 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:26:28,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39710 deadline: 1732710448368, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:28,372 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:28,372 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:28,372 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:28,372 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:26:28,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39698 deadline: 1732710448368, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:28,373 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:28,373 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:28,373 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:26:28,680 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:28,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39696 deadline: 1732710448678, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:28,681 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:28,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39710 deadline: 1732710448678, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:28,681 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:28,681 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:28,681 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:26:28,681 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:28,681 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:28,681 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:28,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39698 deadline: 1732710448678, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:28,681 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:26:28,681 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:28,681 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:28,681 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:26:28,685 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=185 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/.tmp/B/caac659266c2481483bed5cf6a3a6ff3 2024-11-27T12:26:28,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/.tmp/C/f595a0d32eb840ee92a88cb6582e0fbe is 50, key is test_row_0/C:col10/1732710386857/Put/seqid=0 2024-11-27T12:26:28,694 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742301_1477 (size=12151) 2024-11-27T12:26:29,094 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=185 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/.tmp/C/f595a0d32eb840ee92a88cb6582e0fbe 2024-11-27T12:26:29,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/.tmp/A/8bb3eacd4e7942e49e9628f358b7bdb0 as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/A/8bb3eacd4e7942e49e9628f358b7bdb0 2024-11-27T12:26:29,100 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/A/8bb3eacd4e7942e49e9628f358b7bdb0, entries=150, sequenceid=185, filesize=11.9 K 2024-11-27T12:26:29,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/.tmp/B/caac659266c2481483bed5cf6a3a6ff3 as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/B/caac659266c2481483bed5cf6a3a6ff3 2024-11-27T12:26:29,103 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/B/caac659266c2481483bed5cf6a3a6ff3, entries=150, sequenceid=185, filesize=11.9 K 2024-11-27T12:26:29,104 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/.tmp/C/f595a0d32eb840ee92a88cb6582e0fbe as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/C/f595a0d32eb840ee92a88cb6582e0fbe 2024-11-27T12:26:29,107 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/C/f595a0d32eb840ee92a88cb6582e0fbe, entries=150, sequenceid=185, filesize=11.9 K 2024-11-27T12:26:29,107 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=87.22 KB/89310 for 3144c5c576970df3a2a1c91af06cc315 in 1306ms, sequenceid=185, compaction requested=false 2024-11-27T12:26:29,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2603): Flush status journal for 3144c5c576970df3a2a1c91af06cc315: 2024-11-27T12:26:29,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315. 2024-11-27T12:26:29,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=136 2024-11-27T12:26:29,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.HMaster(4169): Remote procedure done, pid=136 2024-11-27T12:26:29,110 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=136, resume processing ppid=135 2024-11-27T12:26:29,110 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=136, ppid=135, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9180 sec 2024-11-27T12:26:29,111 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=135, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees in 1.9210 sec 2024-11-27T12:26:29,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(8855): Flush requested on 3144c5c576970df3a2a1c91af06cc315 2024-11-27T12:26:29,193 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T12:26:29,193 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3144c5c576970df3a2a1c91af06cc315 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-11-27T12:26:29,199 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/.tmp/A/c15e584acb854667a3d7ed320daf55c8 is 50, key is test_row_0/A:col10/1732710389192/Put/seqid=0 2024-11-27T12:26:29,203 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742302_1478 (size=16931) 2024-11-27T12:26:29,235 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:29,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39696 deadline: 1732710449224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:29,236 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:29,236 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:29,236 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:29,236 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:26:29,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39698 deadline: 1732710449232, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:29,236 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:29,236 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:29,236 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:29,236 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:26:29,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39710 deadline: 1732710449233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:29,236 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:29,236 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:29,236 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:26:29,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-27T12:26:29,338 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T12:26:29,339 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-27T12:26:29,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] procedure2.ProcedureExecutor(1139): Stored pid=137, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees 2024-11-27T12:26:29,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-27T12:26:29,340 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=137, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T12:26:29,341 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=137, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T12:26:29,341 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=138, ppid=137, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T12:26:29,342 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:29,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39696 deadline: 1732710449338, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:29,343 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:29,343 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:29,343 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:26:29,355 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:29,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39698 deadline: 1732710449348, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:29,355 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:29,355 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:29,355 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:29,355 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:26:29,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39710 deadline: 1732710449348, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:29,355 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:29,355 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:29,355 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:26:29,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-27T12:26:29,493 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-27T12:26:29,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315. 2024-11-27T12:26:29,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315. as already flushing 2024-11-27T12:26:29,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315. 2024-11-27T12:26:29,493 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => 3144c5c576970df3a2a1c91af06cc315, NAME => 'TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:26:29,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => 3144c5c576970df3a2a1c91af06cc315, NAME => 'TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:26:29,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3144c5c576970df3a2a1c91af06cc315, NAME => 'TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3144c5c576970df3a2a1c91af06cc315, NAME => 'TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:26:29,555 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:29,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39696 deadline: 1732710449548, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:29,556 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:29,556 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:29,556 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:26:29,565 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:29,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39698 deadline: 1732710449558, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:29,566 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:29,566 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:29,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39710 deadline: 1732710449558, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:29,566 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:29,566 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:26:29,566 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:29,566 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:29,566 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:26:29,604 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=205 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/.tmp/A/c15e584acb854667a3d7ed320daf55c8 2024-11-27T12:26:29,610 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/.tmp/B/d9c44a2d791e4ee1bfaf4d5c192e4a64 is 50, key is test_row_0/B:col10/1732710389192/Put/seqid=0 2024-11-27T12:26:29,613 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742303_1479 (size=12151) 2024-11-27T12:26:29,645 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-27T12:26:29,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315. 2024-11-27T12:26:29,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315. as already flushing 2024-11-27T12:26:29,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315. 2024-11-27T12:26:29,645 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => 3144c5c576970df3a2a1c91af06cc315, NAME => 'TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:26:29,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => 3144c5c576970df3a2a1c91af06cc315, NAME => 'TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:26:29,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3144c5c576970df3a2a1c91af06cc315, NAME => 'TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3144c5c576970df3a2a1c91af06cc315, NAME => 'TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:26:29,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-27T12:26:29,798 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-27T12:26:29,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315. 2024-11-27T12:26:29,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315. as already flushing 2024-11-27T12:26:29,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315. 2024-11-27T12:26:29,798 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => 3144c5c576970df3a2a1c91af06cc315, NAME => 'TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:26:29,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => 3144c5c576970df3a2a1c91af06cc315, NAME => 'TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:26:29,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3144c5c576970df3a2a1c91af06cc315, NAME => 'TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3144c5c576970df3a2a1c91af06cc315, NAME => 'TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:26:29,875 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:29,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39696 deadline: 1732710449868, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:29,876 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:29,876 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:29,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39698 deadline: 1732710449868, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:29,876 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:29,876 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:26:29,876 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:29,876 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:29,876 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:26:29,885 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:29,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39710 deadline: 1732710449878, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:29,885 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:29,885 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:29,885 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:26:29,950 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-27T12:26:29,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315. 2024-11-27T12:26:29,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315. as already flushing 2024-11-27T12:26:29,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315. 2024-11-27T12:26:29,951 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => 3144c5c576970df3a2a1c91af06cc315, NAME => 'TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:26:29,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => 3144c5c576970df3a2a1c91af06cc315, NAME => 'TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:26:29,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3144c5c576970df3a2a1c91af06cc315, NAME => 'TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3144c5c576970df3a2a1c91af06cc315, NAME => 'TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:26:29,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-27T12:26:30,014 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=205 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/.tmp/B/d9c44a2d791e4ee1bfaf4d5c192e4a64 2024-11-27T12:26:30,015 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-27T12:26:30,020 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/.tmp/C/2208e1ff36ce449fa6916bcd53534723 is 50, key is test_row_0/C:col10/1732710389192/Put/seqid=0 2024-11-27T12:26:30,023 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742304_1480 (size=12151) 2024-11-27T12:26:30,103 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-27T12:26:30,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315. 2024-11-27T12:26:30,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315. as already flushing 2024-11-27T12:26:30,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315. 2024-11-27T12:26:30,104 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => 3144c5c576970df3a2a1c91af06cc315, NAME => 'TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:26:30,104 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => 3144c5c576970df3a2a1c91af06cc315, NAME => 'TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:26:30,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3144c5c576970df3a2a1c91af06cc315, NAME => 'TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3144c5c576970df3a2a1c91af06cc315, NAME => 'TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:26:30,256 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-27T12:26:30,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315. 2024-11-27T12:26:30,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315. as already flushing 2024-11-27T12:26:30,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315. 2024-11-27T12:26:30,256 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => 3144c5c576970df3a2a1c91af06cc315, NAME => 'TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:26:30,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => 3144c5c576970df3a2a1c91af06cc315, NAME => 'TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:26:30,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3144c5c576970df3a2a1c91af06cc315, NAME => 'TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3144c5c576970df3a2a1c91af06cc315, NAME => 'TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:26:30,384 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:30,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39696 deadline: 1732710450378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:30,385 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:30,385 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:30,385 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:26:30,394 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:30,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39698 deadline: 1732710450388, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:30,395 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:30,395 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:30,395 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:30,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39710 deadline: 1732710450388, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:30,395 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:26:30,395 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:30,395 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:30,395 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:26:30,408 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-27T12:26:30,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315. 2024-11-27T12:26:30,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315. as already flushing 2024-11-27T12:26:30,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315. 2024-11-27T12:26:30,409 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => 3144c5c576970df3a2a1c91af06cc315, NAME => 'TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:26:30,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => 3144c5c576970df3a2a1c91af06cc315, NAME => 'TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:26:30,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3144c5c576970df3a2a1c91af06cc315, NAME => 'TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3144c5c576970df3a2a1c91af06cc315, NAME => 'TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:26:30,423 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=205 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/.tmp/C/2208e1ff36ce449fa6916bcd53534723 2024-11-27T12:26:30,427 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/.tmp/A/c15e584acb854667a3d7ed320daf55c8 as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/A/c15e584acb854667a3d7ed320daf55c8 2024-11-27T12:26:30,429 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/A/c15e584acb854667a3d7ed320daf55c8, entries=250, sequenceid=205, filesize=16.5 K 2024-11-27T12:26:30,430 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/.tmp/B/d9c44a2d791e4ee1bfaf4d5c192e4a64 as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/B/d9c44a2d791e4ee1bfaf4d5c192e4a64 2024-11-27T12:26:30,433 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/B/d9c44a2d791e4ee1bfaf4d5c192e4a64, entries=150, sequenceid=205, filesize=11.9 K 2024-11-27T12:26:30,433 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/.tmp/C/2208e1ff36ce449fa6916bcd53534723 as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/C/2208e1ff36ce449fa6916bcd53534723 2024-11-27T12:26:30,436 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/C/2208e1ff36ce449fa6916bcd53534723, entries=150, sequenceid=205, filesize=11.9 K 2024-11-27T12:26:30,437 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=100.63 KB/103050 for 3144c5c576970df3a2a1c91af06cc315 in 1243ms, sequenceid=205, compaction requested=true 2024-11-27T12:26:30,437 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3144c5c576970df3a2a1c91af06cc315: 2024-11-27T12:26:30,437 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3144c5c576970df3a2a1c91af06cc315:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T12:26:30,437 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T12:26:30,437 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T12:26:30,437 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T12:26:30,437 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3144c5c576970df3a2a1c91af06cc315:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T12:26:30,437 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T12:26:30,437 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3144c5c576970df3a2a1c91af06cc315:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T12:26:30,437 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T12:26:30,437 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T12:26:30,437 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 41575 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T12:26:30,437 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HStore(1541): 3144c5c576970df3a2a1c91af06cc315/B is initiating minor compaction (all files) 2024-11-27T12:26:30,438 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HStore(1541): 3144c5c576970df3a2a1c91af06cc315/A is initiating minor compaction (all files) 2024-11-27T12:26:30,438 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3144c5c576970df3a2a1c91af06cc315/B in TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315. 2024-11-27T12:26:30,438 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3144c5c576970df3a2a1c91af06cc315/A in TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315. 2024-11-27T12:26:30,438 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/B/d862d616b87640f386465b3289452b22, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/B/caac659266c2481483bed5cf6a3a6ff3, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/B/d9c44a2d791e4ee1bfaf4d5c192e4a64] into tmpdir=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/.tmp, totalSize=35.9 K 2024-11-27T12:26:30,438 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/A/98c249cc1bde47f1b096c9a662048116, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/A/8bb3eacd4e7942e49e9628f358b7bdb0, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/A/c15e584acb854667a3d7ed320daf55c8] into tmpdir=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/.tmp, totalSize=40.6 K 2024-11-27T12:26:30,438 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.Compactor(225): Compacting d862d616b87640f386465b3289452b22, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=164, earliestPutTs=1732710385656 2024-11-27T12:26:30,438 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] compactions.Compactor(225): Compacting 98c249cc1bde47f1b096c9a662048116, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=164, earliestPutTs=1732710385656 2024-11-27T12:26:30,438 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.Compactor(225): Compacting caac659266c2481483bed5cf6a3a6ff3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=185, earliestPutTs=1732710386850 2024-11-27T12:26:30,438 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8bb3eacd4e7942e49e9628f358b7bdb0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=185, earliestPutTs=1732710386850 2024-11-27T12:26:30,438 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.Compactor(225): Compacting d9c44a2d791e4ee1bfaf4d5c192e4a64, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=205, earliestPutTs=1732710388033 2024-11-27T12:26:30,438 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] compactions.Compactor(225): Compacting c15e584acb854667a3d7ed320daf55c8, keycount=250, bloomtype=ROW, size=16.5 K, encoding=NONE, compression=NONE, seqNum=205, earliestPutTs=1732710388033 2024-11-27T12:26:30,445 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3144c5c576970df3a2a1c91af06cc315#A#compaction#401 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 53.85 MB/second 2024-11-27T12:26:30,445 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/.tmp/A/80134e7ec99f437eb81bddea70ab5483 is 50, key is test_row_0/A:col10/1732710389192/Put/seqid=0 2024-11-27T12:26:30,446 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3144c5c576970df3a2a1c91af06cc315#B#compaction#402 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-27T12:26:30,446 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/.tmp/B/d1e4fae9fbe641e6bb8e85ee1903a6b7 is 50, key is test_row_0/B:col10/1732710389192/Put/seqid=0 2024-11-27T12:26:30,456 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742305_1481 (size=12595) 2024-11-27T12:26:30,457 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742306_1482 (size=12595) 2024-11-27T12:26:30,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-27T12:26:30,561 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-27T12:26:30,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315. 2024-11-27T12:26:30,562 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2902): Flushing 3144c5c576970df3a2a1c91af06cc315 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-11-27T12:26:30,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/.tmp/A/cf802a9a2415455183cb799656405776 is 50, key is test_row_0/A:col10/1732710389230/Put/seqid=0 2024-11-27T12:26:30,571 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742307_1483 (size=12151) 2024-11-27T12:26:30,860 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/.tmp/A/80134e7ec99f437eb81bddea70ab5483 as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/A/80134e7ec99f437eb81bddea70ab5483 2024-11-27T12:26:30,860 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/.tmp/B/d1e4fae9fbe641e6bb8e85ee1903a6b7 as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/B/d1e4fae9fbe641e6bb8e85ee1903a6b7 2024-11-27T12:26:30,864 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3144c5c576970df3a2a1c91af06cc315/B of 3144c5c576970df3a2a1c91af06cc315 into d1e4fae9fbe641e6bb8e85ee1903a6b7(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T12:26:30,864 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3144c5c576970df3a2a1c91af06cc315: 2024-11-27T12:26:30,864 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., storeName=3144c5c576970df3a2a1c91af06cc315/B, priority=13, startTime=1732710390437; duration=0sec 2024-11-27T12:26:30,864 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T12:26:30,864 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3144c5c576970df3a2a1c91af06cc315:B 2024-11-27T12:26:30,864 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T12:26:30,864 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3144c5c576970df3a2a1c91af06cc315/A of 3144c5c576970df3a2a1c91af06cc315 into 80134e7ec99f437eb81bddea70ab5483(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T12:26:30,864 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3144c5c576970df3a2a1c91af06cc315: 2024-11-27T12:26:30,864 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., storeName=3144c5c576970df3a2a1c91af06cc315/A, priority=13, startTime=1732710390437; duration=0sec 2024-11-27T12:26:30,864 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T12:26:30,864 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3144c5c576970df3a2a1c91af06cc315:A 2024-11-27T12:26:30,865 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T12:26:30,865 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HStore(1541): 3144c5c576970df3a2a1c91af06cc315/C is initiating minor compaction (all files) 2024-11-27T12:26:30,865 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3144c5c576970df3a2a1c91af06cc315/C in TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315. 2024-11-27T12:26:30,865 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/C/7dd9abb37bea4c45b38854bae479c2ae, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/C/f595a0d32eb840ee92a88cb6582e0fbe, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/C/2208e1ff36ce449fa6916bcd53534723] into tmpdir=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/.tmp, totalSize=35.9 K 2024-11-27T12:26:30,865 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.Compactor(225): Compacting 7dd9abb37bea4c45b38854bae479c2ae, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=164, earliestPutTs=1732710385656 2024-11-27T12:26:30,865 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.Compactor(225): Compacting f595a0d32eb840ee92a88cb6582e0fbe, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=185, earliestPutTs=1732710386850 2024-11-27T12:26:30,866 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.Compactor(225): Compacting 2208e1ff36ce449fa6916bcd53534723, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=205, earliestPutTs=1732710388033 2024-11-27T12:26:30,870 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3144c5c576970df3a2a1c91af06cc315#C#compaction#404 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-27T12:26:30,871 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/.tmp/C/40f8c7350e824ae1b61f49dea7e10abd is 50, key is test_row_0/C:col10/1732710389192/Put/seqid=0 2024-11-27T12:26:30,873 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742308_1484 (size=12595) 2024-11-27T12:26:30,878 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/.tmp/C/40f8c7350e824ae1b61f49dea7e10abd as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/C/40f8c7350e824ae1b61f49dea7e10abd 2024-11-27T12:26:30,881 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3144c5c576970df3a2a1c91af06cc315/C of 3144c5c576970df3a2a1c91af06cc315 into 40f8c7350e824ae1b61f49dea7e10abd(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T12:26:30,881 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3144c5c576970df3a2a1c91af06cc315: 2024-11-27T12:26:30,881 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., storeName=3144c5c576970df3a2a1c91af06cc315/C, priority=13, startTime=1732710390437; duration=0sec 2024-11-27T12:26:30,881 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T12:26:30,881 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3144c5c576970df3a2a1c91af06cc315:C 2024-11-27T12:26:30,972 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=223 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/.tmp/A/cf802a9a2415455183cb799656405776 2024-11-27T12:26:30,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/.tmp/B/559395d336544256928d2336c5c60d0a is 50, key is test_row_0/B:col10/1732710389230/Put/seqid=0 2024-11-27T12:26:30,988 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742309_1485 (size=12151) 2024-11-27T12:26:30,989 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=223 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/.tmp/B/559395d336544256928d2336c5c60d0a 2024-11-27T12:26:30,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/.tmp/C/ed88643264c6485db7d05eaa6e27da2b is 50, key is test_row_0/C:col10/1732710389230/Put/seqid=0 2024-11-27T12:26:30,998 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742310_1486 (size=12151) 2024-11-27T12:26:31,398 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=223 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/.tmp/C/ed88643264c6485db7d05eaa6e27da2b 2024-11-27T12:26:31,401 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315. as already flushing 2024-11-27T12:26:31,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(8855): Flush requested on 3144c5c576970df3a2a1c91af06cc315 2024-11-27T12:26:31,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/.tmp/A/cf802a9a2415455183cb799656405776 as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/A/cf802a9a2415455183cb799656405776 2024-11-27T12:26:31,405 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/A/cf802a9a2415455183cb799656405776, entries=150, sequenceid=223, filesize=11.9 K 2024-11-27T12:26:31,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/.tmp/B/559395d336544256928d2336c5c60d0a as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/B/559395d336544256928d2336c5c60d0a 2024-11-27T12:26:31,409 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/B/559395d336544256928d2336c5c60d0a, entries=150, sequenceid=223, filesize=11.9 K 2024-11-27T12:26:31,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/.tmp/C/ed88643264c6485db7d05eaa6e27da2b as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/C/ed88643264c6485db7d05eaa6e27da2b 2024-11-27T12:26:31,413 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/C/ed88643264c6485db7d05eaa6e27da2b, entries=150, sequenceid=223, filesize=11.9 K 2024-11-27T12:26:31,414 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=40.25 KB/41220 for 3144c5c576970df3a2a1c91af06cc315 in 853ms, sequenceid=223, compaction requested=false 2024-11-27T12:26:31,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2603): Flush status journal for 3144c5c576970df3a2a1c91af06cc315: 2024-11-27T12:26:31,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315. 2024-11-27T12:26:31,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=138 2024-11-27T12:26:31,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.HMaster(4169): Remote procedure done, pid=138 2024-11-27T12:26:31,417 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=138, resume processing ppid=137 2024-11-27T12:26:31,417 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=138, ppid=137, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0740 sec 2024-11-27T12:26:31,418 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=137, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees in 2.0780 sec 2024-11-27T12:26:31,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(8855): Flush requested on 3144c5c576970df3a2a1c91af06cc315 2024-11-27T12:26:31,419 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T12:26:31,419 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3144c5c576970df3a2a1c91af06cc315 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-27T12:26:31,423 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/.tmp/A/8bb0c4772b504adf8ff6369ae7ab4f7e is 50, key is test_row_0/A:col10/1732710391418/Put/seqid=0 2024-11-27T12:26:31,426 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742311_1487 (size=14541) 2024-11-27T12:26:31,474 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:31,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39698 deadline: 1732710451464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:31,474 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:31,474 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:31,474 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:31,474 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:26:31,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39710 deadline: 1732710451464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:31,474 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:31,474 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:31,474 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:31,475 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:26:31,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39696 deadline: 1732710451465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:31,475 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:31,475 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:31,475 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:26:31,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-27T12:26:31,498 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T12:26:31,499 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-27T12:26:31,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] procedure2.ProcedureExecutor(1139): Stored pid=139, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=139, table=TestAcidGuarantees 2024-11-27T12:26:31,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-27T12:26:31,500 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=139, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=139, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T12:26:31,501 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=139, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=139, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T12:26:31,501 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=140, ppid=139, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T12:26:31,583 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:31,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39698 deadline: 1732710451578, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:31,584 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:31,584 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:31,584 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:31,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39710 deadline: 1732710451578, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:31,584 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:26:31,584 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:31,584 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:31,584 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:31,584 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:26:31,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39696 deadline: 1732710451578, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:31,584 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:31,584 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:31,584 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:26:31,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-27T12:26:31,653 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-11-27T12:26:31,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315. 2024-11-27T12:26:31,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315. as already flushing 2024-11-27T12:26:31,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315. 2024-11-27T12:26:31,653 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] handler.RSProcedureHandler(58): pid=140 java.io.IOException: Unable to complete flush {ENCODED => 3144c5c576970df3a2a1c91af06cc315, NAME => 'TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:26:31,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=140 java.io.IOException: Unable to complete flush {ENCODED => 3144c5c576970df3a2a1c91af06cc315, NAME => 'TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:26:31,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.HMaster(4177): Remote procedure failed, pid=140 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3144c5c576970df3a2a1c91af06cc315, NAME => 'TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3144c5c576970df3a2a1c91af06cc315, NAME => 'TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:26:31,792 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:31,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39698 deadline: 1732710451788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:31,792 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:31,792 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:31,792 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:31,792 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:26:31,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39710 deadline: 1732710451788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:31,793 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:31,793 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:31,793 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:26:31,793 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:31,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39696 deadline: 1732710451788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:31,793 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:31,793 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:31,793 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:26:31,805 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-11-27T12:26:31,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315. 2024-11-27T12:26:31,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315. as already flushing 2024-11-27T12:26:31,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315. 2024-11-27T12:26:31,806 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] handler.RSProcedureHandler(58): pid=140 java.io.IOException: Unable to complete flush {ENCODED => 3144c5c576970df3a2a1c91af06cc315, NAME => 'TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:26:31,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=140 java.io.IOException: Unable to complete flush {ENCODED => 3144c5c576970df3a2a1c91af06cc315, NAME => 'TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:26:31,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.HMaster(4177): Remote procedure failed, pid=140 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3144c5c576970df3a2a1c91af06cc315, NAME => 'TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3144c5c576970df3a2a1c91af06cc315, NAME => 'TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:26:31,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-27T12:26:31,827 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=237 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/.tmp/A/8bb0c4772b504adf8ff6369ae7ab4f7e 2024-11-27T12:26:31,832 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/.tmp/B/31a0b657386f49a7a2ba634dae686d31 is 50, key is test_row_0/B:col10/1732710391418/Put/seqid=0 2024-11-27T12:26:31,835 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742312_1488 (size=12151) 2024-11-27T12:26:31,958 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-11-27T12:26:31,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315. 2024-11-27T12:26:31,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315. as already flushing 2024-11-27T12:26:31,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315. 2024-11-27T12:26:31,958 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] handler.RSProcedureHandler(58): pid=140 java.io.IOException: Unable to complete flush {ENCODED => 3144c5c576970df3a2a1c91af06cc315, NAME => 'TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:26:31,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=140 java.io.IOException: Unable to complete flush {ENCODED => 3144c5c576970df3a2a1c91af06cc315, NAME => 'TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:26:31,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.HMaster(4177): Remote procedure failed, pid=140 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3144c5c576970df3a2a1c91af06cc315, NAME => 'TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3144c5c576970df3a2a1c91af06cc315, NAME => 'TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:26:32,104 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:32,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39698 deadline: 1732710452098, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:32,105 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:32,105 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:32,105 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:32,105 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:26:32,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39710 deadline: 1732710452098, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:32,105 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:32,105 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:32,105 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:32,105 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:26:32,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39696 deadline: 1732710452099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:32,105 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:32,105 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:32,105 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:26:32,111 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-11-27T12:26:32,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315. 2024-11-27T12:26:32,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315. as already flushing 2024-11-27T12:26:32,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315. 2024-11-27T12:26:32,111 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] handler.RSProcedureHandler(58): pid=140 java.io.IOException: Unable to complete flush {ENCODED => 3144c5c576970df3a2a1c91af06cc315, NAME => 'TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:26:32,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=140 java.io.IOException: Unable to complete flush {ENCODED => 3144c5c576970df3a2a1c91af06cc315, NAME => 'TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:26:32,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.HMaster(4177): Remote procedure failed, pid=140 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3144c5c576970df3a2a1c91af06cc315, NAME => 'TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3144c5c576970df3a2a1c91af06cc315, NAME => 'TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:26:32,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-27T12:26:32,235 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=237 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/.tmp/B/31a0b657386f49a7a2ba634dae686d31 2024-11-27T12:26:32,240 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/.tmp/C/4d0b53bcc56b4f0aadef81262443eaf6 is 50, key is test_row_0/C:col10/1732710391418/Put/seqid=0 2024-11-27T12:26:32,243 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742313_1489 (size=12151) 2024-11-27T12:26:32,263 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-11-27T12:26:32,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315. 2024-11-27T12:26:32,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315. as already flushing 2024-11-27T12:26:32,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315. 2024-11-27T12:26:32,263 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] handler.RSProcedureHandler(58): pid=140 java.io.IOException: Unable to complete flush {ENCODED => 3144c5c576970df3a2a1c91af06cc315, NAME => 'TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:26:32,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=140 java.io.IOException: Unable to complete flush {ENCODED => 3144c5c576970df3a2a1c91af06cc315, NAME => 'TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:26:32,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.HMaster(4177): Remote procedure failed, pid=140 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3144c5c576970df3a2a1c91af06cc315, NAME => 'TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3144c5c576970df3a2a1c91af06cc315, NAME => 'TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:26:32,416 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-11-27T12:26:32,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315. 2024-11-27T12:26:32,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315. as already flushing 2024-11-27T12:26:32,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315. 2024-11-27T12:26:32,416 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] handler.RSProcedureHandler(58): pid=140 java.io.IOException: Unable to complete flush {ENCODED => 3144c5c576970df3a2a1c91af06cc315, NAME => 'TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:26:32,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=140 java.io.IOException: Unable to complete flush {ENCODED => 3144c5c576970df3a2a1c91af06cc315, NAME => 'TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:26:32,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.HMaster(4177): Remote procedure failed, pid=140 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3144c5c576970df3a2a1c91af06cc315, NAME => 'TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3144c5c576970df3a2a1c91af06cc315, NAME => 'TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:26:32,569 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-11-27T12:26:32,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315. 2024-11-27T12:26:32,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315. as already flushing 2024-11-27T12:26:32,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315. 2024-11-27T12:26:32,569 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] handler.RSProcedureHandler(58): pid=140 java.io.IOException: Unable to complete flush {ENCODED => 3144c5c576970df3a2a1c91af06cc315, NAME => 'TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:26:32,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=140 java.io.IOException: Unable to complete flush {ENCODED => 3144c5c576970df3a2a1c91af06cc315, NAME => 'TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:26:32,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.HMaster(4177): Remote procedure failed, pid=140 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3144c5c576970df3a2a1c91af06cc315, NAME => 'TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3144c5c576970df3a2a1c91af06cc315, NAME => 'TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:26:32,611 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:32,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39698 deadline: 1732710452608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:32,611 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:32,611 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:32,612 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:32,612 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:26:32,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39710 deadline: 1732710452608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:32,612 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:32,612 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:32,612 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:26:32,620 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:32,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39696 deadline: 1732710452617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:32,621 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:32,621 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:32,621 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:26:32,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-27T12:26:32,644 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=237 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/.tmp/C/4d0b53bcc56b4f0aadef81262443eaf6 2024-11-27T12:26:32,647 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/.tmp/A/8bb0c4772b504adf8ff6369ae7ab4f7e as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/A/8bb0c4772b504adf8ff6369ae7ab4f7e 2024-11-27T12:26:32,650 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/A/8bb0c4772b504adf8ff6369ae7ab4f7e, entries=200, sequenceid=237, filesize=14.2 K 2024-11-27T12:26:32,651 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/.tmp/B/31a0b657386f49a7a2ba634dae686d31 as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/B/31a0b657386f49a7a2ba634dae686d31 2024-11-27T12:26:32,653 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/B/31a0b657386f49a7a2ba634dae686d31, entries=150, sequenceid=237, filesize=11.9 K 2024-11-27T12:26:32,654 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/.tmp/C/4d0b53bcc56b4f0aadef81262443eaf6 as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/C/4d0b53bcc56b4f0aadef81262443eaf6 2024-11-27T12:26:32,656 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/C/4d0b53bcc56b4f0aadef81262443eaf6, entries=150, sequenceid=237, filesize=11.9 K 2024-11-27T12:26:32,657 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 3144c5c576970df3a2a1c91af06cc315 in 1238ms, sequenceid=237, compaction requested=true 2024-11-27T12:26:32,657 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3144c5c576970df3a2a1c91af06cc315: 2024-11-27T12:26:32,657 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3144c5c576970df3a2a1c91af06cc315:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T12:26:32,657 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T12:26:32,657 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T12:26:32,658 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3144c5c576970df3a2a1c91af06cc315:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T12:26:32,658 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T12:26:32,658 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T12:26:32,658 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3144c5c576970df3a2a1c91af06cc315:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T12:26:32,658 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T12:26:32,658 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39287 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T12:26:32,658 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T12:26:32,658 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HStore(1541): 3144c5c576970df3a2a1c91af06cc315/A is initiating minor compaction (all files) 2024-11-27T12:26:32,658 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HStore(1541): 3144c5c576970df3a2a1c91af06cc315/B is initiating minor compaction (all files) 2024-11-27T12:26:32,658 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3144c5c576970df3a2a1c91af06cc315/A in TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315. 2024-11-27T12:26:32,658 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3144c5c576970df3a2a1c91af06cc315/B in TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315. 2024-11-27T12:26:32,658 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/A/80134e7ec99f437eb81bddea70ab5483, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/A/cf802a9a2415455183cb799656405776, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/A/8bb0c4772b504adf8ff6369ae7ab4f7e] into tmpdir=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/.tmp, totalSize=38.4 K 2024-11-27T12:26:32,658 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/B/d1e4fae9fbe641e6bb8e85ee1903a6b7, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/B/559395d336544256928d2336c5c60d0a, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/B/31a0b657386f49a7a2ba634dae686d31] into tmpdir=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/.tmp, totalSize=36.0 K 2024-11-27T12:26:32,659 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] compactions.Compactor(225): Compacting 80134e7ec99f437eb81bddea70ab5483, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=205, earliestPutTs=1732710388033 2024-11-27T12:26:32,659 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.Compactor(225): Compacting d1e4fae9fbe641e6bb8e85ee1903a6b7, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=205, earliestPutTs=1732710388033 2024-11-27T12:26:32,659 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] compactions.Compactor(225): Compacting cf802a9a2415455183cb799656405776, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=223, earliestPutTs=1732710389221 2024-11-27T12:26:32,659 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.Compactor(225): Compacting 559395d336544256928d2336c5c60d0a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=223, earliestPutTs=1732710389221 2024-11-27T12:26:32,659 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.Compactor(225): Compacting 31a0b657386f49a7a2ba634dae686d31, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=237, earliestPutTs=1732710391412 2024-11-27T12:26:32,659 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8bb0c4772b504adf8ff6369ae7ab4f7e, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=237, earliestPutTs=1732710391411 2024-11-27T12:26:32,665 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3144c5c576970df3a2a1c91af06cc315#A#compaction#410 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 53.85 MB/second 2024-11-27T12:26:32,665 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/.tmp/A/cf05d92764f441a8b8f92102a2694362 is 50, key is test_row_0/A:col10/1732710391418/Put/seqid=0 2024-11-27T12:26:32,665 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3144c5c576970df3a2a1c91af06cc315#B#compaction#411 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-27T12:26:32,666 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/.tmp/B/bda50c2896f64eaf99a39bd3e0bf753b is 50, key is test_row_0/B:col10/1732710391418/Put/seqid=0 2024-11-27T12:26:32,673 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742314_1490 (size=12697) 2024-11-27T12:26:32,678 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742315_1491 (size=12697) 2024-11-27T12:26:32,721 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-11-27T12:26:32,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315. 2024-11-27T12:26:32,722 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2902): Flushing 3144c5c576970df3a2a1c91af06cc315 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-27T12:26:32,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/.tmp/A/af9017fae64d4c258239ca87d412a157 is 50, key is test_row_0/A:col10/1732710391464/Put/seqid=0 2024-11-27T12:26:32,730 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742316_1492 (size=12251) 2024-11-27T12:26:32,731 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=262 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/.tmp/A/af9017fae64d4c258239ca87d412a157 2024-11-27T12:26:32,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/.tmp/B/848c1f78a57848209eb904f5a81da8af is 50, key is test_row_0/B:col10/1732710391464/Put/seqid=0 2024-11-27T12:26:32,740 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742317_1493 (size=12251) 2024-11-27T12:26:33,114 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/.tmp/A/cf05d92764f441a8b8f92102a2694362 as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/A/cf05d92764f441a8b8f92102a2694362 2024-11-27T12:26:33,116 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/.tmp/B/bda50c2896f64eaf99a39bd3e0bf753b as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/B/bda50c2896f64eaf99a39bd3e0bf753b 2024-11-27T12:26:33,118 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3144c5c576970df3a2a1c91af06cc315/A of 3144c5c576970df3a2a1c91af06cc315 into cf05d92764f441a8b8f92102a2694362(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T12:26:33,118 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3144c5c576970df3a2a1c91af06cc315: 2024-11-27T12:26:33,118 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., storeName=3144c5c576970df3a2a1c91af06cc315/A, priority=13, startTime=1732710392657; duration=0sec 2024-11-27T12:26:33,118 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T12:26:33,118 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3144c5c576970df3a2a1c91af06cc315:A 2024-11-27T12:26:33,118 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T12:26:33,119 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T12:26:33,119 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HStore(1541): 3144c5c576970df3a2a1c91af06cc315/C is initiating minor compaction (all files) 2024-11-27T12:26:33,119 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3144c5c576970df3a2a1c91af06cc315/C in TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315. 2024-11-27T12:26:33,119 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/C/40f8c7350e824ae1b61f49dea7e10abd, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/C/ed88643264c6485db7d05eaa6e27da2b, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/C/4d0b53bcc56b4f0aadef81262443eaf6] into tmpdir=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/.tmp, totalSize=36.0 K 2024-11-27T12:26:33,119 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] compactions.Compactor(225): Compacting 40f8c7350e824ae1b61f49dea7e10abd, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=205, earliestPutTs=1732710388033 2024-11-27T12:26:33,120 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] compactions.Compactor(225): Compacting ed88643264c6485db7d05eaa6e27da2b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=223, earliestPutTs=1732710389221 2024-11-27T12:26:33,120 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3144c5c576970df3a2a1c91af06cc315/B of 3144c5c576970df3a2a1c91af06cc315 into bda50c2896f64eaf99a39bd3e0bf753b(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T12:26:33,120 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3144c5c576970df3a2a1c91af06cc315: 2024-11-27T12:26:33,120 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., storeName=3144c5c576970df3a2a1c91af06cc315/B, priority=13, startTime=1732710392658; duration=0sec 2024-11-27T12:26:33,120 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T12:26:33,120 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3144c5c576970df3a2a1c91af06cc315:B 2024-11-27T12:26:33,120 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4d0b53bcc56b4f0aadef81262443eaf6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=237, earliestPutTs=1732710391412 2024-11-27T12:26:33,125 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3144c5c576970df3a2a1c91af06cc315#C#compaction#414 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-27T12:26:33,125 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/.tmp/C/a918db7de0314ec5ac65f60bf2964c84 is 50, key is test_row_0/C:col10/1732710391418/Put/seqid=0 2024-11-27T12:26:33,128 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742318_1494 (size=12697) 2024-11-27T12:26:33,141 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=262 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/.tmp/B/848c1f78a57848209eb904f5a81da8af 2024-11-27T12:26:33,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/.tmp/C/e482a28d44c842358857d715d860bb9e is 50, key is test_row_0/C:col10/1732710391464/Put/seqid=0 2024-11-27T12:26:33,151 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742319_1495 (size=12251) 2024-11-27T12:26:33,532 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/.tmp/C/a918db7de0314ec5ac65f60bf2964c84 as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/C/a918db7de0314ec5ac65f60bf2964c84 2024-11-27T12:26:33,536 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3144c5c576970df3a2a1c91af06cc315/C of 3144c5c576970df3a2a1c91af06cc315 into a918db7de0314ec5ac65f60bf2964c84(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T12:26:33,536 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3144c5c576970df3a2a1c91af06cc315: 2024-11-27T12:26:33,536 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., storeName=3144c5c576970df3a2a1c91af06cc315/C, priority=13, startTime=1732710392658; duration=0sec 2024-11-27T12:26:33,536 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T12:26:33,536 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3144c5c576970df3a2a1c91af06cc315:C 2024-11-27T12:26:33,553 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=262 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/.tmp/C/e482a28d44c842358857d715d860bb9e 2024-11-27T12:26:33,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/.tmp/A/af9017fae64d4c258239ca87d412a157 as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/A/af9017fae64d4c258239ca87d412a157 2024-11-27T12:26:33,558 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/A/af9017fae64d4c258239ca87d412a157, entries=150, sequenceid=262, filesize=12.0 K 2024-11-27T12:26:33,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/.tmp/B/848c1f78a57848209eb904f5a81da8af as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/B/848c1f78a57848209eb904f5a81da8af 2024-11-27T12:26:33,561 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/B/848c1f78a57848209eb904f5a81da8af, entries=150, sequenceid=262, filesize=12.0 K 2024-11-27T12:26:33,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/.tmp/C/e482a28d44c842358857d715d860bb9e as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/C/e482a28d44c842358857d715d860bb9e 2024-11-27T12:26:33,565 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/C/e482a28d44c842358857d715d860bb9e, entries=150, sequenceid=262, filesize=12.0 K 2024-11-27T12:26:33,565 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=0 B/0 for 3144c5c576970df3a2a1c91af06cc315 in 843ms, sequenceid=262, compaction requested=false 2024-11-27T12:26:33,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2603): Flush status journal for 3144c5c576970df3a2a1c91af06cc315: 2024-11-27T12:26:33,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315. 2024-11-27T12:26:33,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=140 2024-11-27T12:26:33,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.HMaster(4169): Remote procedure done, pid=140 2024-11-27T12:26:33,568 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=140, resume processing ppid=139 2024-11-27T12:26:33,568 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=140, ppid=139, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0650 sec 2024-11-27T12:26:33,570 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=139, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=139, table=TestAcidGuarantees in 2.0700 sec 2024-11-27T12:26:33,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(8855): Flush requested on 3144c5c576970df3a2a1c91af06cc315 2024-11-27T12:26:33,629 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T12:26:33,629 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3144c5c576970df3a2a1c91af06cc315 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-27T12:26:33,633 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/.tmp/A/50117f45792441e7a4f18634ce737e57 is 50, key is test_row_0/A:col10/1732710393628/Put/seqid=0 2024-11-27T12:26:33,636 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742320_1496 (size=17181) 2024-11-27T12:26:33,638 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=276 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/.tmp/A/50117f45792441e7a4f18634ce737e57 2024-11-27T12:26:33,643 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/.tmp/B/f5bc3f6796a747ef8885c3e6385f652f is 50, key is test_row_0/B:col10/1732710393628/Put/seqid=0 2024-11-27T12:26:33,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-27T12:26:33,648 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T12:26:33,648 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-27T12:26:33,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] procedure2.ProcedureExecutor(1139): Stored pid=141, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=141, table=TestAcidGuarantees 2024-11-27T12:26:33,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-27T12:26:33,650 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=141, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=141, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T12:26:33,650 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=141, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=141, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T12:26:33,650 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=142, ppid=141, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T12:26:33,651 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742321_1497 (size=12301) 2024-11-27T12:26:33,716 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:33,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39710 deadline: 1732710453714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:33,716 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:33,716 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:33,716 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:26:33,722 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:33,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39698 deadline: 1732710453715, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:33,723 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:33,723 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:33,723 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:26:33,724 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:33,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39696 deadline: 1732710453716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:33,724 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:33,724 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:33,724 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:26:33,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-27T12:26:33,803 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=142 2024-11-27T12:26:33,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315. 2024-11-27T12:26:33,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315. as already flushing 2024-11-27T12:26:33,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315. 2024-11-27T12:26:33,803 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] handler.RSProcedureHandler(58): pid=142 java.io.IOException: Unable to complete flush {ENCODED => 3144c5c576970df3a2a1c91af06cc315, NAME => 'TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:26:33,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=142 java.io.IOException: Unable to complete flush {ENCODED => 3144c5c576970df3a2a1c91af06cc315, NAME => 'TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:26:33,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.HMaster(4177): Remote procedure failed, pid=142 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3144c5c576970df3a2a1c91af06cc315, NAME => 'TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3144c5c576970df3a2a1c91af06cc315, NAME => 'TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:26:33,824 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:33,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39710 deadline: 1732710453818, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:33,824 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:33,824 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:33,825 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:26:33,833 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:33,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39698 deadline: 1732710453828, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:33,834 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:33,834 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:33,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39696 deadline: 1732710453828, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:33,834 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:33,834 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:26:33,834 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:33,834 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:33,834 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:26:33,955 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=142 2024-11-27T12:26:33,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315. 2024-11-27T12:26:33,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315. as already flushing 2024-11-27T12:26:33,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315. 2024-11-27T12:26:33,956 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] handler.RSProcedureHandler(58): pid=142 java.io.IOException: Unable to complete flush {ENCODED => 3144c5c576970df3a2a1c91af06cc315, NAME => 'TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:26:33,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=142 java.io.IOException: Unable to complete flush {ENCODED => 3144c5c576970df3a2a1c91af06cc315, NAME => 'TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:26:33,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.HMaster(4177): Remote procedure failed, pid=142 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3144c5c576970df3a2a1c91af06cc315, NAME => 'TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3144c5c576970df3a2a1c91af06cc315, NAME => 'TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:26:33,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-27T12:26:34,034 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:34,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39710 deadline: 1732710454028, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:34,035 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:34,035 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:34,035 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:26:34,044 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:34,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39698 deadline: 1732710454038, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:34,044 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:34,044 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:34,044 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:26:34,044 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:34,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39696 deadline: 1732710454038, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:34,045 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:34,045 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:34,045 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:26:34,052 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=276 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/.tmp/B/f5bc3f6796a747ef8885c3e6385f652f 2024-11-27T12:26:34,057 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/.tmp/C/2d8d5c8c2d88417f8d9cc618cc56fb41 is 50, key is test_row_0/C:col10/1732710393628/Put/seqid=0 2024-11-27T12:26:34,061 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742322_1498 (size=12301) 2024-11-27T12:26:34,108 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=142 2024-11-27T12:26:34,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315. 2024-11-27T12:26:34,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315. as already flushing 2024-11-27T12:26:34,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315. 2024-11-27T12:26:34,109 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] handler.RSProcedureHandler(58): pid=142 java.io.IOException: Unable to complete flush {ENCODED => 3144c5c576970df3a2a1c91af06cc315, NAME => 'TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:26:34,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=142 java.io.IOException: Unable to complete flush {ENCODED => 3144c5c576970df3a2a1c91af06cc315, NAME => 'TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:26:34,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.HMaster(4177): Remote procedure failed, pid=142 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3144c5c576970df3a2a1c91af06cc315, NAME => 'TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3144c5c576970df3a2a1c91af06cc315, NAME => 'TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:26:34,261 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=142 2024-11-27T12:26:34,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315. 2024-11-27T12:26:34,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315. as already flushing 2024-11-27T12:26:34,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315. 2024-11-27T12:26:34,261 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] handler.RSProcedureHandler(58): pid=142 java.io.IOException: Unable to complete flush {ENCODED => 3144c5c576970df3a2a1c91af06cc315, NAME => 'TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:26:34,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=142 java.io.IOException: Unable to complete flush {ENCODED => 3144c5c576970df3a2a1c91af06cc315, NAME => 'TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:26:34,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.HMaster(4177): Remote procedure failed, pid=142 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3144c5c576970df3a2a1c91af06cc315, NAME => 'TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3144c5c576970df3a2a1c91af06cc315, NAME => 'TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:26:34,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-27T12:26:34,345 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:34,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39710 deadline: 1732710454338, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:34,345 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:34,345 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:34,345 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:26:34,354 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:34,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39698 deadline: 1732710454348, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:34,354 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:34,354 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:34,354 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:34,354 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:26:34,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39696 deadline: 1732710454348, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:34,354 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:34,354 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:34,354 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:26:34,414 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=142 2024-11-27T12:26:34,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315. 2024-11-27T12:26:34,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315. as already flushing 2024-11-27T12:26:34,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315. 2024-11-27T12:26:34,414 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] handler.RSProcedureHandler(58): pid=142 java.io.IOException: Unable to complete flush {ENCODED => 3144c5c576970df3a2a1c91af06cc315, NAME => 'TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:26:34,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=142 java.io.IOException: Unable to complete flush {ENCODED => 3144c5c576970df3a2a1c91af06cc315, NAME => 'TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:26:34,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.HMaster(4177): Remote procedure failed, pid=142 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3144c5c576970df3a2a1c91af06cc315, NAME => 'TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3144c5c576970df3a2a1c91af06cc315, NAME => 'TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:26:34,462 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=276 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/.tmp/C/2d8d5c8c2d88417f8d9cc618cc56fb41 2024-11-27T12:26:34,465 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/.tmp/A/50117f45792441e7a4f18634ce737e57 as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/A/50117f45792441e7a4f18634ce737e57 2024-11-27T12:26:34,468 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/A/50117f45792441e7a4f18634ce737e57, entries=250, sequenceid=276, filesize=16.8 K 2024-11-27T12:26:34,468 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/.tmp/B/f5bc3f6796a747ef8885c3e6385f652f as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/B/f5bc3f6796a747ef8885c3e6385f652f 2024-11-27T12:26:34,470 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/B/f5bc3f6796a747ef8885c3e6385f652f, entries=150, sequenceid=276, filesize=12.0 K 2024-11-27T12:26:34,471 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/.tmp/C/2d8d5c8c2d88417f8d9cc618cc56fb41 as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/C/2d8d5c8c2d88417f8d9cc618cc56fb41 2024-11-27T12:26:34,474 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/C/2d8d5c8c2d88417f8d9cc618cc56fb41, entries=150, sequenceid=276, filesize=12.0 K 2024-11-27T12:26:34,474 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 3144c5c576970df3a2a1c91af06cc315 in 845ms, sequenceid=276, compaction requested=true 2024-11-27T12:26:34,475 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3144c5c576970df3a2a1c91af06cc315: 2024-11-27T12:26:34,475 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3144c5c576970df3a2a1c91af06cc315:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T12:26:34,475 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T12:26:34,475 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T12:26:34,475 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3144c5c576970df3a2a1c91af06cc315:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T12:26:34,475 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T12:26:34,475 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T12:26:34,475 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3144c5c576970df3a2a1c91af06cc315:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T12:26:34,475 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T12:26:34,475 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37249 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T12:26:34,475 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42129 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T12:26:34,475 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HStore(1541): 3144c5c576970df3a2a1c91af06cc315/A is initiating minor compaction (all files) 2024-11-27T12:26:34,475 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HStore(1541): 3144c5c576970df3a2a1c91af06cc315/B is initiating minor compaction (all files) 2024-11-27T12:26:34,475 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3144c5c576970df3a2a1c91af06cc315/B in TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315. 2024-11-27T12:26:34,475 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3144c5c576970df3a2a1c91af06cc315/A in TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315. 2024-11-27T12:26:34,476 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/B/bda50c2896f64eaf99a39bd3e0bf753b, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/B/848c1f78a57848209eb904f5a81da8af, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/B/f5bc3f6796a747ef8885c3e6385f652f] into tmpdir=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/.tmp, totalSize=36.4 K 2024-11-27T12:26:34,476 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/A/cf05d92764f441a8b8f92102a2694362, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/A/af9017fae64d4c258239ca87d412a157, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/A/50117f45792441e7a4f18634ce737e57] into tmpdir=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/.tmp, totalSize=41.1 K 2024-11-27T12:26:34,476 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.Compactor(225): Compacting bda50c2896f64eaf99a39bd3e0bf753b, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=237, earliestPutTs=1732710391412 2024-11-27T12:26:34,476 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] compactions.Compactor(225): Compacting cf05d92764f441a8b8f92102a2694362, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=237, earliestPutTs=1732710391412 2024-11-27T12:26:34,476 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.Compactor(225): Compacting 848c1f78a57848209eb904f5a81da8af, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=262, earliestPutTs=1732710391451 2024-11-27T12:26:34,476 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] compactions.Compactor(225): Compacting af9017fae64d4c258239ca87d412a157, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=262, earliestPutTs=1732710391451 2024-11-27T12:26:34,476 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.Compactor(225): Compacting f5bc3f6796a747ef8885c3e6385f652f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=276, earliestPutTs=1732710393618 2024-11-27T12:26:34,476 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] compactions.Compactor(225): Compacting 50117f45792441e7a4f18634ce737e57, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=276, earliestPutTs=1732710393618 2024-11-27T12:26:34,484 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3144c5c576970df3a2a1c91af06cc315#A#compaction#419 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-27T12:26:34,484 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/.tmp/A/153a89f9dcc946ec8ae672243e25a44f is 50, key is test_row_0/A:col10/1732710393628/Put/seqid=0 2024-11-27T12:26:34,494 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3144c5c576970df3a2a1c91af06cc315#B#compaction#420 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-27T12:26:34,495 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/.tmp/B/d9d1376d0f2a4cf89cbc12b4ebfc0f23 is 50, key is test_row_0/B:col10/1732710393628/Put/seqid=0 2024-11-27T12:26:34,507 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742323_1499 (size=12949) 2024-11-27T12:26:34,513 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742324_1500 (size=12949) 2024-11-27T12:26:34,566 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=142 2024-11-27T12:26:34,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315. 2024-11-27T12:26:34,567 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegion(2902): Flushing 3144c5c576970df3a2a1c91af06cc315 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-27T12:26:34,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/.tmp/A/938c08518fe94ff986bdf58ccfb40873 is 50, key is test_row_0/A:col10/1732710393712/Put/seqid=0 2024-11-27T12:26:34,581 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742325_1501 (size=12301) 2024-11-27T12:26:34,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-27T12:26:34,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(8855): Flush requested on 3144c5c576970df3a2a1c91af06cc315 2024-11-27T12:26:34,863 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315. as already flushing 2024-11-27T12:26:34,883 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:34,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39696 deadline: 1732710454874, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:34,883 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:34,883 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:34,883 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:34,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39698 deadline: 1732710454875, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:34,884 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:26:34,884 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:34,884 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:34,884 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:26:34,885 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:34,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39710 deadline: 1732710454876, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:34,885 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:34,885 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:34,885 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:26:34,911 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/.tmp/A/153a89f9dcc946ec8ae672243e25a44f as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/A/153a89f9dcc946ec8ae672243e25a44f 2024-11-27T12:26:34,915 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3144c5c576970df3a2a1c91af06cc315/A of 3144c5c576970df3a2a1c91af06cc315 into 153a89f9dcc946ec8ae672243e25a44f(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T12:26:34,915 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3144c5c576970df3a2a1c91af06cc315: 2024-11-27T12:26:34,915 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., storeName=3144c5c576970df3a2a1c91af06cc315/A, priority=13, startTime=1732710394475; duration=0sec 2024-11-27T12:26:34,915 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T12:26:34,915 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3144c5c576970df3a2a1c91af06cc315:A 2024-11-27T12:26:34,915 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T12:26:34,917 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37249 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T12:26:34,917 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HStore(1541): 3144c5c576970df3a2a1c91af06cc315/C is initiating minor compaction (all files) 2024-11-27T12:26:34,917 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3144c5c576970df3a2a1c91af06cc315/C in TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315. 2024-11-27T12:26:34,917 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/C/a918db7de0314ec5ac65f60bf2964c84, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/C/e482a28d44c842358857d715d860bb9e, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/C/2d8d5c8c2d88417f8d9cc618cc56fb41] into tmpdir=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/.tmp, totalSize=36.4 K 2024-11-27T12:26:34,918 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] compactions.Compactor(225): Compacting a918db7de0314ec5ac65f60bf2964c84, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=237, earliestPutTs=1732710391412 2024-11-27T12:26:34,918 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] compactions.Compactor(225): Compacting e482a28d44c842358857d715d860bb9e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=262, earliestPutTs=1732710391451 2024-11-27T12:26:34,918 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/.tmp/B/d9d1376d0f2a4cf89cbc12b4ebfc0f23 as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/B/d9d1376d0f2a4cf89cbc12b4ebfc0f23 2024-11-27T12:26:34,918 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2d8d5c8c2d88417f8d9cc618cc56fb41, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=276, earliestPutTs=1732710393618 2024-11-27T12:26:34,922 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3144c5c576970df3a2a1c91af06cc315/B of 3144c5c576970df3a2a1c91af06cc315 into d9d1376d0f2a4cf89cbc12b4ebfc0f23(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T12:26:34,922 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3144c5c576970df3a2a1c91af06cc315: 2024-11-27T12:26:34,922 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., storeName=3144c5c576970df3a2a1c91af06cc315/B, priority=13, startTime=1732710394475; duration=0sec 2024-11-27T12:26:34,922 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T12:26:34,922 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3144c5c576970df3a2a1c91af06cc315:B 2024-11-27T12:26:34,925 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3144c5c576970df3a2a1c91af06cc315#C#compaction#422 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-27T12:26:34,925 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/.tmp/C/d2cd42089f9d4b77b846612f55bac727 is 50, key is test_row_0/C:col10/1732710393628/Put/seqid=0 2024-11-27T12:26:34,930 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742326_1502 (size=12949) 2024-11-27T12:26:34,982 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=301 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/.tmp/A/938c08518fe94ff986bdf58ccfb40873 2024-11-27T12:26:34,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/.tmp/B/e263b33973a04e3e9ef28c5fe47b0944 is 50, key is test_row_0/B:col10/1732710393712/Put/seqid=0 2024-11-27T12:26:34,992 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:34,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39696 deadline: 1732710454988, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:34,992 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:34,992 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:34,992 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:34,992 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:26:34,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39698 deadline: 1732710454988, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:34,993 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:34,993 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:34,993 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:34,993 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:26:34,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39710 deadline: 1732710454988, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:34,993 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:34,993 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:34,993 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:26:34,995 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742327_1503 (size=12301) 2024-11-27T12:26:35,200 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:35,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39696 deadline: 1732710455198, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:35,200 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:35,200 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:35,200 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:35,200 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:26:35,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39698 deadline: 1732710455198, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:35,201 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:35,201 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:35,201 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:26:35,201 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:35,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39710 deadline: 1732710455198, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:35,201 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:35,201 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:35,201 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:26:35,334 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/.tmp/C/d2cd42089f9d4b77b846612f55bac727 as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/C/d2cd42089f9d4b77b846612f55bac727 2024-11-27T12:26:35,338 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3144c5c576970df3a2a1c91af06cc315/C of 3144c5c576970df3a2a1c91af06cc315 into d2cd42089f9d4b77b846612f55bac727(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T12:26:35,338 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3144c5c576970df3a2a1c91af06cc315: 2024-11-27T12:26:35,338 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., storeName=3144c5c576970df3a2a1c91af06cc315/C, priority=13, startTime=1732710394475; duration=0sec 2024-11-27T12:26:35,338 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T12:26:35,338 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3144c5c576970df3a2a1c91af06cc315:C 2024-11-27T12:26:35,396 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=301 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/.tmp/B/e263b33973a04e3e9ef28c5fe47b0944 2024-11-27T12:26:35,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/.tmp/C/8bb1bab37c97414bb84d0ffdc4d7c599 is 50, key is test_row_0/C:col10/1732710393712/Put/seqid=0 2024-11-27T12:26:35,405 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742328_1504 (size=12301) 2024-11-27T12:26:35,514 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:35,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39696 deadline: 1732710455508, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:35,514 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:35,514 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:35,514 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:35,514 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:26:35,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39698 deadline: 1732710455508, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:35,515 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:35,515 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:35,515 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:35,515 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:26:35,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39710 deadline: 1732710455509, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:35,515 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:35,515 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:35,515 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:26:35,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-27T12:26:35,806 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=301 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/.tmp/C/8bb1bab37c97414bb84d0ffdc4d7c599 2024-11-27T12:26:35,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/.tmp/A/938c08518fe94ff986bdf58ccfb40873 as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/A/938c08518fe94ff986bdf58ccfb40873 2024-11-27T12:26:35,812 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/A/938c08518fe94ff986bdf58ccfb40873, entries=150, sequenceid=301, filesize=12.0 K 2024-11-27T12:26:35,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/.tmp/B/e263b33973a04e3e9ef28c5fe47b0944 as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/B/e263b33973a04e3e9ef28c5fe47b0944 2024-11-27T12:26:35,815 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/B/e263b33973a04e3e9ef28c5fe47b0944, entries=150, sequenceid=301, filesize=12.0 K 2024-11-27T12:26:35,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/.tmp/C/8bb1bab37c97414bb84d0ffdc4d7c599 as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/C/8bb1bab37c97414bb84d0ffdc4d7c599 2024-11-27T12:26:35,818 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/C/8bb1bab37c97414bb84d0ffdc4d7c599, entries=150, sequenceid=301, filesize=12.0 K 2024-11-27T12:26:35,819 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 3144c5c576970df3a2a1c91af06cc315 in 1252ms, sequenceid=301, compaction requested=false 2024-11-27T12:26:35,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegion(2603): Flush status journal for 3144c5c576970df3a2a1c91af06cc315: 2024-11-27T12:26:35,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315. 2024-11-27T12:26:35,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=142 2024-11-27T12:26:35,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.HMaster(4169): Remote procedure done, pid=142 2024-11-27T12:26:35,822 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=142, resume processing ppid=141 2024-11-27T12:26:35,822 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=142, ppid=141, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1700 sec 2024-11-27T12:26:35,823 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=141, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=141, table=TestAcidGuarantees in 2.1740 sec 2024-11-27T12:26:36,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(8855): Flush requested on 3144c5c576970df3a2a1c91af06cc315 2024-11-27T12:26:36,020 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T12:26:36,020 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3144c5c576970df3a2a1c91af06cc315 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-27T12:26:36,024 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/.tmp/A/eb7516065e9a4ac19df641958a247a59 is 50, key is test_row_0/A:col10/1732710394875/Put/seqid=0 2024-11-27T12:26:36,028 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742329_1505 (size=14741) 2024-11-27T12:26:36,074 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:36,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39696 deadline: 1732710456073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:36,075 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:36,075 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:36,075 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:26:36,083 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:36,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39698 deadline: 1732710456074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:36,083 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:36,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39710 deadline: 1732710456074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:36,083 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:36,083 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:36,083 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:26:36,085 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:36,085 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:36,085 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:26:36,184 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:36,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39696 deadline: 1732710456178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:36,185 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:36,185 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:36,185 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:26:36,194 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:36,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39698 deadline: 1732710456188, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:36,194 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:36,194 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:36,194 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:36,194 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:26:36,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39710 deadline: 1732710456188, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:36,194 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:36,194 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:36,194 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:26:36,392 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:36,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39696 deadline: 1732710456388, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:36,392 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:36,392 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:36,392 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:26:36,401 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:36,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39698 deadline: 1732710456398, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:36,402 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:36,402 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:36,402 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:36,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39710 deadline: 1732710456398, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:36,402 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:26:36,402 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:36,402 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:36,402 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:26:36,428 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=316 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/.tmp/A/eb7516065e9a4ac19df641958a247a59 2024-11-27T12:26:36,435 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/.tmp/B/59a287f5e0794b16bdb0ae0c6b9fcd66 is 50, key is test_row_0/B:col10/1732710394875/Put/seqid=0 2024-11-27T12:26:36,446 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742330_1506 (size=12301) 2024-11-27T12:26:36,704 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:36,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39696 deadline: 1732710456698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:36,704 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:36,704 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:36,704 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:26:36,714 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:36,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39698 deadline: 1732710456708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:36,714 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:36,714 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:36,714 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:36,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39710 deadline: 1732710456708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:36,714 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:26:36,714 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:36,715 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:36,715 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:26:36,847 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=316 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/.tmp/B/59a287f5e0794b16bdb0ae0c6b9fcd66 2024-11-27T12:26:36,852 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/.tmp/C/634736fb747d4b71ae7a14a43ce80fca is 50, key is test_row_0/C:col10/1732710394875/Put/seqid=0 2024-11-27T12:26:36,855 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742331_1507 (size=12301) 2024-11-27T12:26:36,915 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:36,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39712 deadline: 1732710456908, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:36,916 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 1ba5af43fd99,36251,1732710241631 for 'test_row_1' in 3144c5c576970df3a2a1c91af06cc315 of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18242 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T12:26:36,916 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:36,916 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:36,916 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:26:37,044 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:37,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39674 deadline: 1732710457038, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:37,045 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 1ba5af43fd99,36251,1732710241631 for 'test_row_2' in 3144c5c576970df3a2a1c91af06cc315 of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18371 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T12:26:37,045 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:37,045 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:37,045 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:26:37,215 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:37,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39696 deadline: 1732710457208, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:37,215 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:37,215 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:37,215 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:26:37,233 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:37,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39698 deadline: 1732710457228, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:37,234 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:37,234 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:37,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39710 deadline: 1732710457228, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:37,234 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:37,234 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:26:37,234 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 , the old value is region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:37,234 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:37,234 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., hostname=1ba5af43fd99,36251,1732710241631, seqNum=2 because the exception is null or not the one we care about 2024-11-27T12:26:37,256 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=316 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/.tmp/C/634736fb747d4b71ae7a14a43ce80fca 2024-11-27T12:26:37,259 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/.tmp/A/eb7516065e9a4ac19df641958a247a59 as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/A/eb7516065e9a4ac19df641958a247a59 2024-11-27T12:26:37,262 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/A/eb7516065e9a4ac19df641958a247a59, entries=200, sequenceid=316, filesize=14.4 K 2024-11-27T12:26:37,263 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/.tmp/B/59a287f5e0794b16bdb0ae0c6b9fcd66 as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/B/59a287f5e0794b16bdb0ae0c6b9fcd66 2024-11-27T12:26:37,266 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/B/59a287f5e0794b16bdb0ae0c6b9fcd66, entries=150, sequenceid=316, filesize=12.0 K 2024-11-27T12:26:37,266 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/.tmp/C/634736fb747d4b71ae7a14a43ce80fca as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/C/634736fb747d4b71ae7a14a43ce80fca 2024-11-27T12:26:37,269 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/C/634736fb747d4b71ae7a14a43ce80fca, entries=150, sequenceid=316, filesize=12.0 K 2024-11-27T12:26:37,270 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 3144c5c576970df3a2a1c91af06cc315 in 1250ms, sequenceid=316, compaction requested=true 2024-11-27T12:26:37,270 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3144c5c576970df3a2a1c91af06cc315: 2024-11-27T12:26:37,270 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3144c5c576970df3a2a1c91af06cc315:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T12:26:37,270 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T12:26:37,270 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T12:26:37,270 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3144c5c576970df3a2a1c91af06cc315:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T12:26:37,270 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T12:26:37,270 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3144c5c576970df3a2a1c91af06cc315:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T12:26:37,270 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T12:26:37,270 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T12:26:37,271 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39991 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T12:26:37,271 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T12:26:37,271 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HStore(1541): 3144c5c576970df3a2a1c91af06cc315/A is initiating minor compaction (all files) 2024-11-27T12:26:37,271 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HStore(1541): 3144c5c576970df3a2a1c91af06cc315/B is initiating minor compaction (all files) 2024-11-27T12:26:37,271 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3144c5c576970df3a2a1c91af06cc315/A in TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315. 2024-11-27T12:26:37,271 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3144c5c576970df3a2a1c91af06cc315/B in TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315. 2024-11-27T12:26:37,271 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/A/153a89f9dcc946ec8ae672243e25a44f, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/A/938c08518fe94ff986bdf58ccfb40873, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/A/eb7516065e9a4ac19df641958a247a59] into tmpdir=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/.tmp, totalSize=39.1 K 2024-11-27T12:26:37,271 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/B/d9d1376d0f2a4cf89cbc12b4ebfc0f23, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/B/e263b33973a04e3e9ef28c5fe47b0944, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/B/59a287f5e0794b16bdb0ae0c6b9fcd66] into tmpdir=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/.tmp, totalSize=36.7 K 2024-11-27T12:26:37,271 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.Compactor(225): Compacting d9d1376d0f2a4cf89cbc12b4ebfc0f23, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=276, earliestPutTs=1732710393618 2024-11-27T12:26:37,271 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] compactions.Compactor(225): Compacting 153a89f9dcc946ec8ae672243e25a44f, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=276, earliestPutTs=1732710393618 2024-11-27T12:26:37,271 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.Compactor(225): Compacting e263b33973a04e3e9ef28c5fe47b0944, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=301, earliestPutTs=1732710393712 2024-11-27T12:26:37,272 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] compactions.Compactor(225): Compacting 938c08518fe94ff986bdf58ccfb40873, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=301, earliestPutTs=1732710393712 2024-11-27T12:26:37,272 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.Compactor(225): Compacting 59a287f5e0794b16bdb0ae0c6b9fcd66, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=316, earliestPutTs=1732710394874 2024-11-27T12:26:37,272 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] compactions.Compactor(225): Compacting eb7516065e9a4ac19df641958a247a59, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=316, earliestPutTs=1732710394865 2024-11-27T12:26:37,277 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3144c5c576970df3a2a1c91af06cc315#B#compaction#428 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 53.85 MB/second 2024-11-27T12:26:37,278 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3144c5c576970df3a2a1c91af06cc315#A#compaction#429 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-27T12:26:37,278 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/.tmp/B/c5d3885675dc45feab0f2e94dd38b96a is 50, key is test_row_0/B:col10/1732710394875/Put/seqid=0 2024-11-27T12:26:37,278 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/.tmp/A/155f608cb45c435984034fb0d77fc00f is 50, key is test_row_0/A:col10/1732710394875/Put/seqid=0 2024-11-27T12:26:37,282 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742333_1509 (size=13051) 2024-11-27T12:26:37,289 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742332_1508 (size=13051) 2024-11-27T12:26:37,315 INFO [Thread-1999 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1999. 2024-11-27T12:26:37,315 INFO [Thread-1995 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1995. 2024-11-27T12:26:37,315 DEBUG [Thread-1999 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T12:26:37,315 DEBUG [Thread-1999 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T12:26:37,315 DEBUG [Thread-1995 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T12:26:37,315 DEBUG [Thread-1995 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T12:26:37,315 DEBUG [Thread-1999 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T12:26:37,315 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T12:26:37,316 DEBUG [Thread-1995 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T12:26:37,316 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T12:26:37,318 INFO [Thread-2001 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2001. 2024-11-27T12:26:37,318 DEBUG [Thread-2001 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T12:26:37,318 DEBUG [Thread-2001 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T12:26:37,318 DEBUG [Thread-2001 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T12:26:37,318 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T12:26:37,321 INFO [Thread-1993 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1993. 2024-11-27T12:26:37,321 DEBUG [Thread-1993 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T12:26:37,321 DEBUG [Thread-1993 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T12:26:37,321 DEBUG [Thread-1993 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T12:26:37,321 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T12:26:37,323 INFO [Thread-1997 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1997. 2024-11-27T12:26:37,323 DEBUG [Thread-1997 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T12:26:37,323 DEBUG [Thread-1997 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T12:26:37,323 DEBUG [Thread-1997 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T12:26:37,323 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T12:26:37,693 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/.tmp/A/155f608cb45c435984034fb0d77fc00f as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/A/155f608cb45c435984034fb0d77fc00f 2024-11-27T12:26:37,695 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/.tmp/B/c5d3885675dc45feab0f2e94dd38b96a as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/B/c5d3885675dc45feab0f2e94dd38b96a 2024-11-27T12:26:37,699 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3144c5c576970df3a2a1c91af06cc315/A of 3144c5c576970df3a2a1c91af06cc315 into 155f608cb45c435984034fb0d77fc00f(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T12:26:37,699 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3144c5c576970df3a2a1c91af06cc315: 2024-11-27T12:26:37,699 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., storeName=3144c5c576970df3a2a1c91af06cc315/A, priority=13, startTime=1732710397270; duration=0sec 2024-11-27T12:26:37,699 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T12:26:37,699 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3144c5c576970df3a2a1c91af06cc315:A 2024-11-27T12:26:37,699 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T12:26:37,700 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T12:26:37,700 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HStore(1541): 3144c5c576970df3a2a1c91af06cc315/C is initiating minor compaction (all files) 2024-11-27T12:26:37,700 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3144c5c576970df3a2a1c91af06cc315/C in TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315. 2024-11-27T12:26:37,700 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/C/d2cd42089f9d4b77b846612f55bac727, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/C/8bb1bab37c97414bb84d0ffdc4d7c599, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/C/634736fb747d4b71ae7a14a43ce80fca] into tmpdir=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/.tmp, totalSize=36.7 K 2024-11-27T12:26:37,701 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] compactions.Compactor(225): Compacting d2cd42089f9d4b77b846612f55bac727, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=276, earliestPutTs=1732710393618 2024-11-27T12:26:37,701 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3144c5c576970df3a2a1c91af06cc315/B of 3144c5c576970df3a2a1c91af06cc315 into c5d3885675dc45feab0f2e94dd38b96a(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T12:26:37,701 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3144c5c576970df3a2a1c91af06cc315: 2024-11-27T12:26:37,701 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., storeName=3144c5c576970df3a2a1c91af06cc315/B, priority=13, startTime=1732710397270; duration=0sec 2024-11-27T12:26:37,701 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8bb1bab37c97414bb84d0ffdc4d7c599, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=301, earliestPutTs=1732710393712 2024-11-27T12:26:37,701 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T12:26:37,701 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3144c5c576970df3a2a1c91af06cc315:B 2024-11-27T12:26:37,701 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] compactions.Compactor(225): Compacting 634736fb747d4b71ae7a14a43ce80fca, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=316, earliestPutTs=1732710394874 2024-11-27T12:26:37,709 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3144c5c576970df3a2a1c91af06cc315#C#compaction#430 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-27T12:26:37,709 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/.tmp/C/6680f24e696d428cb28ce242d25a2789 is 50, key is test_row_0/C:col10/1732710394875/Put/seqid=0 2024-11-27T12:26:37,713 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742334_1510 (size=13051) 2024-11-27T12:26:37,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-27T12:26:37,820 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T12:26:38,149 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/.tmp/C/6680f24e696d428cb28ce242d25a2789 as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/C/6680f24e696d428cb28ce242d25a2789 2024-11-27T12:26:38,156 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3144c5c576970df3a2a1c91af06cc315/C of 3144c5c576970df3a2a1c91af06cc315 into 6680f24e696d428cb28ce242d25a2789(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T12:26:38,156 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3144c5c576970df3a2a1c91af06cc315: 2024-11-27T12:26:38,156 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., storeName=3144c5c576970df3a2a1c91af06cc315/C, priority=13, startTime=1732710397270; duration=0sec 2024-11-27T12:26:38,156 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T12:26:38,156 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3144c5c576970df3a2a1c91af06cc315:C 2024-11-27T12:26:38,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(8855): Flush requested on 3144c5c576970df3a2a1c91af06cc315 2024-11-27T12:26:38,229 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T12:26:38,229 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3144c5c576970df3a2a1c91af06cc315 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-27T12:26:38,230 INFO [Thread-1984 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1984. 2024-11-27T12:26:38,230 DEBUG [Thread-1984 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T12:26:38,230 DEBUG [Thread-1984 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T12:26:38,230 DEBUG [Thread-1984 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T12:26:38,230 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T12:26:38,234 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/.tmp/A/f02e108fcab140cca46169e85f99c840 is 50, key is test_row_0/A:col10/1732710396073/Put/seqid=0 2024-11-27T12:26:38,238 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742335_1511 (size=12301) 2024-11-27T12:26:38,240 INFO [Thread-1986 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1986. 2024-11-27T12:26:38,240 INFO [Thread-1988 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1988. 2024-11-27T12:26:38,240 DEBUG [Thread-1986 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T12:26:38,240 DEBUG [Thread-1986 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T12:26:38,240 DEBUG [Thread-1988 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T12:26:38,240 DEBUG [Thread-1988 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T12:26:38,240 DEBUG [Thread-1986 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T12:26:38,240 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T12:26:38,241 DEBUG [Thread-1988 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T12:26:38,241 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T12:26:38,640 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=344 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/.tmp/A/f02e108fcab140cca46169e85f99c840 2024-11-27T12:26:38,657 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/.tmp/B/d992f90bba4e4813852be714c476a2c0 is 50, key is test_row_0/B:col10/1732710396073/Put/seqid=0 2024-11-27T12:26:38,660 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742336_1512 (size=12301) 2024-11-27T12:26:39,062 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=344 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/.tmp/B/d992f90bba4e4813852be714c476a2c0 2024-11-27T12:26:39,076 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/.tmp/C/696614b9908b4d918b76e36abb3570fa is 50, key is test_row_0/C:col10/1732710396073/Put/seqid=0 2024-11-27T12:26:39,080 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742337_1513 (size=12301) 2024-11-27T12:26:39,482 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=344 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/.tmp/C/696614b9908b4d918b76e36abb3570fa 2024-11-27T12:26:39,490 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/.tmp/A/f02e108fcab140cca46169e85f99c840 as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/A/f02e108fcab140cca46169e85f99c840 2024-11-27T12:26:39,496 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/A/f02e108fcab140cca46169e85f99c840, entries=150, sequenceid=344, filesize=12.0 K 2024-11-27T12:26:39,497 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/.tmp/B/d992f90bba4e4813852be714c476a2c0 as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/B/d992f90bba4e4813852be714c476a2c0 2024-11-27T12:26:39,502 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/B/d992f90bba4e4813852be714c476a2c0, entries=150, sequenceid=344, filesize=12.0 K 2024-11-27T12:26:39,503 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/.tmp/C/696614b9908b4d918b76e36abb3570fa as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/C/696614b9908b4d918b76e36abb3570fa 2024-11-27T12:26:39,507 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/C/696614b9908b4d918b76e36abb3570fa, entries=150, sequenceid=344, filesize=12.0 K 2024-11-27T12:26:39,508 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=13.42 KB/13740 for 3144c5c576970df3a2a1c91af06cc315 in 1279ms, sequenceid=344, compaction requested=false 2024-11-27T12:26:39,508 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3144c5c576970df3a2a1c91af06cc315: 2024-11-27T12:26:46,963 INFO [Thread-1982 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1982. 2024-11-27T12:26:46,963 DEBUG [Thread-1982 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T12:26:46,963 DEBUG [Thread-1982 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T12:26:46,963 DEBUG [Thread-1982 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T12:26:46,963 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T12:26:47,113 INFO [Thread-1990 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1990. 2024-11-27T12:26:47,113 DEBUG [Thread-1990 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T12:26:47,113 DEBUG [Thread-1990 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T12:26:47,113 DEBUG [Thread-1990 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T12:26:47,114 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T12:26:47,114 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-27T12:26:47,114 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 13 2024-11-27T12:26:47,115 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 76 2024-11-27T12:26:47,115 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 93 2024-11-27T12:26:47,115 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 78 2024-11-27T12:26:47,115 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 15 2024-11-27T12:26:47,115 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-27T12:26:47,115 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-27T12:26:47,115 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1434 2024-11-27T12:26:47,115 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4299 rows 2024-11-27T12:26:47,115 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1432 2024-11-27T12:26:47,115 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4292 rows 2024-11-27T12:26:47,115 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1431 2024-11-27T12:26:47,115 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4291 rows 2024-11-27T12:26:47,115 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1422 2024-11-27T12:26:47,115 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4265 rows 2024-11-27T12:26:47,115 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1432 2024-11-27T12:26:47,115 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4296 rows 2024-11-27T12:26:47,115 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-27T12:26:47,115 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testScanAtomicity(AcidGuaranteesTestBase.java:112) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-27T12:26:47,115 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T12:26:47,116 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T12:26:47,116 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T12:26:47,117 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-11-27T12:26:47,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] procedure2.ProcedureExecutor(1139): Stored pid=143, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-27T12:26:47,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-11-27T12:26:47,123 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732710407122"}]},"ts":"1732710407122"} 2024-11-27T12:26:47,125 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-27T12:26:47,125 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-27T12:26:47,126 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=144, ppid=143, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-27T12:26:47,128 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=145, ppid=144, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=3144c5c576970df3a2a1c91af06cc315, UNASSIGN}] 2024-11-27T12:26:47,129 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=145, ppid=144, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=3144c5c576970df3a2a1c91af06cc315, UNASSIGN 2024-11-27T12:26:47,130 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=145 updating hbase:meta row=3144c5c576970df3a2a1c91af06cc315, regionState=CLOSING, regionLocation=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:47,132 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=145, ppid=144, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=3144c5c576970df3a2a1c91af06cc315, UNASSIGN because future has completed 2024-11-27T12:26:47,132 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-27T12:26:47,132 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=146, ppid=145, state=RUNNABLE, hasLock=false; CloseRegionProcedure 3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631}] 2024-11-27T12:26:47,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-11-27T12:26:47,286 INFO [RS_CLOSE_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] handler.UnassignRegionHandler(122): Close 3144c5c576970df3a2a1c91af06cc315 2024-11-27T12:26:47,286 DEBUG [RS_CLOSE_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-27T12:26:47,286 DEBUG [RS_CLOSE_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegion(1722): Closing 3144c5c576970df3a2a1c91af06cc315, disabling compactions & flushes 2024-11-27T12:26:47,286 INFO [RS_CLOSE_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315. 2024-11-27T12:26:47,286 DEBUG [RS_CLOSE_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315. 2024-11-27T12:26:47,286 DEBUG [RS_CLOSE_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315. after waiting 0 ms 2024-11-27T12:26:47,286 DEBUG [RS_CLOSE_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315. 2024-11-27T12:26:47,287 INFO [RS_CLOSE_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegion(2902): Flushing 3144c5c576970df3a2a1c91af06cc315 3/3 column families, dataSize=26.84 KB heapSize=71.06 KB 2024-11-27T12:26:47,296 DEBUG [RS_CLOSE_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/.tmp/A/e61cabf66c19402689c54dc7ad8f2e03 is 50, key is test_row_1/A:col10/1732710406959/Put/seqid=0 2024-11-27T12:26:47,300 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742338_1514 (size=9857) 2024-11-27T12:26:47,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-11-27T12:26:47,701 INFO [RS_CLOSE_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=351 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/.tmp/A/e61cabf66c19402689c54dc7ad8f2e03 2024-11-27T12:26:47,708 DEBUG [RS_CLOSE_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/.tmp/B/50ebca5376a94d04abb07996432641cb is 50, key is test_row_1/B:col10/1732710406959/Put/seqid=0 2024-11-27T12:26:47,712 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742339_1515 (size=9857) 2024-11-27T12:26:47,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-11-27T12:26:48,114 INFO [RS_CLOSE_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=351 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/.tmp/B/50ebca5376a94d04abb07996432641cb 2024-11-27T12:26:48,125 DEBUG [RS_CLOSE_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/.tmp/C/adf12e0eb20141d0be61e61b698127d6 is 50, key is test_row_1/C:col10/1732710406959/Put/seqid=0 2024-11-27T12:26:48,129 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742340_1516 (size=9857) 2024-11-27T12:26:48,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-11-27T12:26:48,530 INFO [RS_CLOSE_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=351 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/.tmp/C/adf12e0eb20141d0be61e61b698127d6 2024-11-27T12:26:48,540 DEBUG [RS_CLOSE_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/.tmp/A/e61cabf66c19402689c54dc7ad8f2e03 as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/A/e61cabf66c19402689c54dc7ad8f2e03 2024-11-27T12:26:48,546 INFO [RS_CLOSE_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/A/e61cabf66c19402689c54dc7ad8f2e03, entries=100, sequenceid=351, filesize=9.6 K 2024-11-27T12:26:48,547 DEBUG [RS_CLOSE_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/.tmp/B/50ebca5376a94d04abb07996432641cb as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/B/50ebca5376a94d04abb07996432641cb 2024-11-27T12:26:48,551 INFO [RS_CLOSE_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/B/50ebca5376a94d04abb07996432641cb, entries=100, sequenceid=351, filesize=9.6 K 2024-11-27T12:26:48,551 DEBUG [RS_CLOSE_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/.tmp/C/adf12e0eb20141d0be61e61b698127d6 as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/C/adf12e0eb20141d0be61e61b698127d6 2024-11-27T12:26:48,555 INFO [RS_CLOSE_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/C/adf12e0eb20141d0be61e61b698127d6, entries=100, sequenceid=351, filesize=9.6 K 2024-11-27T12:26:48,556 INFO [RS_CLOSE_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~71.02 KB/72720, currentSize=0 B/0 for 3144c5c576970df3a2a1c91af06cc315 in 1270ms, sequenceid=351, compaction requested=true 2024-11-27T12:26:48,557 DEBUG [StoreCloser-TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/A/b1a17dce207041728e6a4cfa67964ffa, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/A/8663aa07aae84174a946123426cd00bf, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/A/3d4b9f70af3144298f9b40975de374bd, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/A/3a0f14592fc7447496497a5e0e21868b, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/A/4426c21656524d38ac158d5c03ca56a8, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/A/78e69fc9eff646f2aa85eb5e7c6cf155, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/A/b8dfae2a2a2e4ab88f18bdd1736c3fa9, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/A/1a8531bdd0574cc2af75b4fa9e98b2b1, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/A/bad67b95594243199cb28fb002eb977e, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/A/98c249cc1bde47f1b096c9a662048116, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/A/d9167162e3a0497bb0ecc171e7887022, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/A/8bb3eacd4e7942e49e9628f358b7bdb0, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/A/c15e584acb854667a3d7ed320daf55c8, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/A/80134e7ec99f437eb81bddea70ab5483, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/A/cf802a9a2415455183cb799656405776, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/A/8bb0c4772b504adf8ff6369ae7ab4f7e, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/A/cf05d92764f441a8b8f92102a2694362, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/A/af9017fae64d4c258239ca87d412a157, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/A/50117f45792441e7a4f18634ce737e57, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/A/153a89f9dcc946ec8ae672243e25a44f, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/A/938c08518fe94ff986bdf58ccfb40873, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/A/eb7516065e9a4ac19df641958a247a59] to archive 2024-11-27T12:26:48,558 DEBUG [StoreCloser-TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-27T12:26:48,560 DEBUG [StoreCloser-TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/A/b1a17dce207041728e6a4cfa67964ffa to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/A/b1a17dce207041728e6a4cfa67964ffa 2024-11-27T12:26:48,561 DEBUG [StoreCloser-TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/A/8663aa07aae84174a946123426cd00bf to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/A/8663aa07aae84174a946123426cd00bf 2024-11-27T12:26:48,562 DEBUG [StoreCloser-TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/A/3d4b9f70af3144298f9b40975de374bd to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/A/3d4b9f70af3144298f9b40975de374bd 2024-11-27T12:26:48,563 DEBUG [StoreCloser-TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/A/3a0f14592fc7447496497a5e0e21868b to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/A/3a0f14592fc7447496497a5e0e21868b 2024-11-27T12:26:48,564 DEBUG [StoreCloser-TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/A/4426c21656524d38ac158d5c03ca56a8 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/A/4426c21656524d38ac158d5c03ca56a8 2024-11-27T12:26:48,565 DEBUG [StoreCloser-TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/A/78e69fc9eff646f2aa85eb5e7c6cf155 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/A/78e69fc9eff646f2aa85eb5e7c6cf155 2024-11-27T12:26:48,566 DEBUG [StoreCloser-TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/A/b8dfae2a2a2e4ab88f18bdd1736c3fa9 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/A/b8dfae2a2a2e4ab88f18bdd1736c3fa9 2024-11-27T12:26:48,567 DEBUG [StoreCloser-TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/A/1a8531bdd0574cc2af75b4fa9e98b2b1 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/A/1a8531bdd0574cc2af75b4fa9e98b2b1 2024-11-27T12:26:48,567 DEBUG [StoreCloser-TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/A/bad67b95594243199cb28fb002eb977e to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/A/bad67b95594243199cb28fb002eb977e 2024-11-27T12:26:48,568 DEBUG [StoreCloser-TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/A/98c249cc1bde47f1b096c9a662048116 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/A/98c249cc1bde47f1b096c9a662048116 2024-11-27T12:26:48,569 DEBUG [StoreCloser-TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/A/d9167162e3a0497bb0ecc171e7887022 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/A/d9167162e3a0497bb0ecc171e7887022 2024-11-27T12:26:48,570 DEBUG [StoreCloser-TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/A/8bb3eacd4e7942e49e9628f358b7bdb0 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/A/8bb3eacd4e7942e49e9628f358b7bdb0 2024-11-27T12:26:48,571 DEBUG [StoreCloser-TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/A/c15e584acb854667a3d7ed320daf55c8 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/A/c15e584acb854667a3d7ed320daf55c8 2024-11-27T12:26:48,571 DEBUG [StoreCloser-TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/A/80134e7ec99f437eb81bddea70ab5483 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/A/80134e7ec99f437eb81bddea70ab5483 2024-11-27T12:26:48,572 DEBUG [StoreCloser-TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/A/cf802a9a2415455183cb799656405776 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/A/cf802a9a2415455183cb799656405776 2024-11-27T12:26:48,573 DEBUG [StoreCloser-TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/A/8bb0c4772b504adf8ff6369ae7ab4f7e to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/A/8bb0c4772b504adf8ff6369ae7ab4f7e 2024-11-27T12:26:48,574 DEBUG [StoreCloser-TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/A/cf05d92764f441a8b8f92102a2694362 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/A/cf05d92764f441a8b8f92102a2694362 2024-11-27T12:26:48,575 DEBUG [StoreCloser-TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/A/af9017fae64d4c258239ca87d412a157 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/A/af9017fae64d4c258239ca87d412a157 2024-11-27T12:26:48,575 DEBUG [StoreCloser-TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/A/50117f45792441e7a4f18634ce737e57 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/A/50117f45792441e7a4f18634ce737e57 2024-11-27T12:26:48,576 DEBUG [StoreCloser-TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/A/153a89f9dcc946ec8ae672243e25a44f to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/A/153a89f9dcc946ec8ae672243e25a44f 2024-11-27T12:26:48,576 DEBUG [StoreCloser-TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/A/938c08518fe94ff986bdf58ccfb40873 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/A/938c08518fe94ff986bdf58ccfb40873 2024-11-27T12:26:48,577 DEBUG [StoreCloser-TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/A/eb7516065e9a4ac19df641958a247a59 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/A/eb7516065e9a4ac19df641958a247a59 2024-11-27T12:26:48,578 DEBUG [StoreCloser-TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/B/eb948be447f54b5eb6b2b773996a6e84, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/B/1eaa295f51194d919d5c4a218f596055, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/B/932d9a74e91e4a69a896780b684453ec, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/B/516c4743ce1a4ea89d0d72b15f4206c8, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/B/708eba06bb944b7c808ec309f8119081, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/B/71dee8d1146f4c8195ab63e5d96e1441, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/B/b7c7b8c43e434a99a7c18e3ca4543307, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/B/fc1ee9fac4964203a81d7ee7c9b76985, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/B/4c7aa0b877704768b07e46e9779b736c, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/B/d862d616b87640f386465b3289452b22, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/B/7fc7d60dd36a4deab80a8d4e046ac1e5, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/B/caac659266c2481483bed5cf6a3a6ff3, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/B/d1e4fae9fbe641e6bb8e85ee1903a6b7, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/B/d9c44a2d791e4ee1bfaf4d5c192e4a64, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/B/559395d336544256928d2336c5c60d0a, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/B/bda50c2896f64eaf99a39bd3e0bf753b, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/B/31a0b657386f49a7a2ba634dae686d31, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/B/848c1f78a57848209eb904f5a81da8af, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/B/d9d1376d0f2a4cf89cbc12b4ebfc0f23, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/B/f5bc3f6796a747ef8885c3e6385f652f, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/B/e263b33973a04e3e9ef28c5fe47b0944, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/B/59a287f5e0794b16bdb0ae0c6b9fcd66] to archive 2024-11-27T12:26:48,579 DEBUG [StoreCloser-TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-27T12:26:48,580 DEBUG [StoreCloser-TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/B/eb948be447f54b5eb6b2b773996a6e84 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/B/eb948be447f54b5eb6b2b773996a6e84 2024-11-27T12:26:48,581 DEBUG [StoreCloser-TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/B/1eaa295f51194d919d5c4a218f596055 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/B/1eaa295f51194d919d5c4a218f596055 2024-11-27T12:26:48,581 DEBUG [StoreCloser-TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/B/932d9a74e91e4a69a896780b684453ec to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/B/932d9a74e91e4a69a896780b684453ec 2024-11-27T12:26:48,582 DEBUG [StoreCloser-TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/B/516c4743ce1a4ea89d0d72b15f4206c8 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/B/516c4743ce1a4ea89d0d72b15f4206c8 2024-11-27T12:26:48,583 DEBUG [StoreCloser-TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/B/708eba06bb944b7c808ec309f8119081 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/B/708eba06bb944b7c808ec309f8119081 2024-11-27T12:26:48,583 DEBUG [StoreCloser-TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/B/71dee8d1146f4c8195ab63e5d96e1441 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/B/71dee8d1146f4c8195ab63e5d96e1441 2024-11-27T12:26:48,584 DEBUG [StoreCloser-TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/B/b7c7b8c43e434a99a7c18e3ca4543307 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/B/b7c7b8c43e434a99a7c18e3ca4543307 2024-11-27T12:26:48,585 DEBUG [StoreCloser-TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/B/fc1ee9fac4964203a81d7ee7c9b76985 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/B/fc1ee9fac4964203a81d7ee7c9b76985 2024-11-27T12:26:48,585 DEBUG [StoreCloser-TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/B/4c7aa0b877704768b07e46e9779b736c to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/B/4c7aa0b877704768b07e46e9779b736c 2024-11-27T12:26:48,586 DEBUG [StoreCloser-TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/B/d862d616b87640f386465b3289452b22 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/B/d862d616b87640f386465b3289452b22 2024-11-27T12:26:48,586 DEBUG [StoreCloser-TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/B/7fc7d60dd36a4deab80a8d4e046ac1e5 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/B/7fc7d60dd36a4deab80a8d4e046ac1e5 2024-11-27T12:26:48,587 DEBUG [StoreCloser-TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/B/caac659266c2481483bed5cf6a3a6ff3 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/B/caac659266c2481483bed5cf6a3a6ff3 2024-11-27T12:26:48,588 DEBUG [StoreCloser-TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/B/d1e4fae9fbe641e6bb8e85ee1903a6b7 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/B/d1e4fae9fbe641e6bb8e85ee1903a6b7 2024-11-27T12:26:48,588 DEBUG [StoreCloser-TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/B/d9c44a2d791e4ee1bfaf4d5c192e4a64 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/B/d9c44a2d791e4ee1bfaf4d5c192e4a64 2024-11-27T12:26:48,589 DEBUG [StoreCloser-TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/B/559395d336544256928d2336c5c60d0a to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/B/559395d336544256928d2336c5c60d0a 2024-11-27T12:26:48,590 DEBUG [StoreCloser-TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/B/bda50c2896f64eaf99a39bd3e0bf753b to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/B/bda50c2896f64eaf99a39bd3e0bf753b 2024-11-27T12:26:48,591 DEBUG [StoreCloser-TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/B/31a0b657386f49a7a2ba634dae686d31 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/B/31a0b657386f49a7a2ba634dae686d31 2024-11-27T12:26:48,591 DEBUG [StoreCloser-TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/B/848c1f78a57848209eb904f5a81da8af to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/B/848c1f78a57848209eb904f5a81da8af 2024-11-27T12:26:48,592 DEBUG [StoreCloser-TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/B/d9d1376d0f2a4cf89cbc12b4ebfc0f23 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/B/d9d1376d0f2a4cf89cbc12b4ebfc0f23 2024-11-27T12:26:48,593 DEBUG [StoreCloser-TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/B/f5bc3f6796a747ef8885c3e6385f652f to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/B/f5bc3f6796a747ef8885c3e6385f652f 2024-11-27T12:26:48,593 DEBUG [StoreCloser-TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/B/e263b33973a04e3e9ef28c5fe47b0944 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/B/e263b33973a04e3e9ef28c5fe47b0944 2024-11-27T12:26:48,594 DEBUG [StoreCloser-TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/B/59a287f5e0794b16bdb0ae0c6b9fcd66 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/B/59a287f5e0794b16bdb0ae0c6b9fcd66 2024-11-27T12:26:48,595 DEBUG [StoreCloser-TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/C/16b0a48d92ae45c5900bdde6eaecfdb2, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/C/ae5edde338334cd4a72f5a131da0389c, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/C/cfff5ebfbf6c40c980f019f20ae46bdb, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/C/cbb1368c48e04278866da73ff790213a, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/C/f61f7010e88441ef8a3b37f74e39c4f6, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/C/6516a82bdf2548daa994dee46975d1be, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/C/6399fa4932bb4b15a6a2c61f9556efb3, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/C/6091c17e2d2e4927902eed2303c2887d, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/C/0007d7093f65428e8372f78cd84daf9b, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/C/7dd9abb37bea4c45b38854bae479c2ae, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/C/de863092a9e544d1bd0a430d366a1772, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/C/f595a0d32eb840ee92a88cb6582e0fbe, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/C/40f8c7350e824ae1b61f49dea7e10abd, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/C/2208e1ff36ce449fa6916bcd53534723, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/C/ed88643264c6485db7d05eaa6e27da2b, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/C/a918db7de0314ec5ac65f60bf2964c84, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/C/4d0b53bcc56b4f0aadef81262443eaf6, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/C/e482a28d44c842358857d715d860bb9e, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/C/d2cd42089f9d4b77b846612f55bac727, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/C/2d8d5c8c2d88417f8d9cc618cc56fb41, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/C/8bb1bab37c97414bb84d0ffdc4d7c599, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/C/634736fb747d4b71ae7a14a43ce80fca] to archive 2024-11-27T12:26:48,595 DEBUG [StoreCloser-TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-27T12:26:48,596 DEBUG [StoreCloser-TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/C/16b0a48d92ae45c5900bdde6eaecfdb2 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/C/16b0a48d92ae45c5900bdde6eaecfdb2 2024-11-27T12:26:48,597 DEBUG [StoreCloser-TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/C/ae5edde338334cd4a72f5a131da0389c to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/C/ae5edde338334cd4a72f5a131da0389c 2024-11-27T12:26:48,598 DEBUG [StoreCloser-TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/C/cfff5ebfbf6c40c980f019f20ae46bdb to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/C/cfff5ebfbf6c40c980f019f20ae46bdb 2024-11-27T12:26:48,598 DEBUG [StoreCloser-TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/C/cbb1368c48e04278866da73ff790213a to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/C/cbb1368c48e04278866da73ff790213a 2024-11-27T12:26:48,599 DEBUG [StoreCloser-TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/C/f61f7010e88441ef8a3b37f74e39c4f6 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/C/f61f7010e88441ef8a3b37f74e39c4f6 2024-11-27T12:26:48,600 DEBUG [StoreCloser-TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/C/6516a82bdf2548daa994dee46975d1be to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/C/6516a82bdf2548daa994dee46975d1be 2024-11-27T12:26:48,600 DEBUG [StoreCloser-TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/C/6399fa4932bb4b15a6a2c61f9556efb3 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/C/6399fa4932bb4b15a6a2c61f9556efb3 2024-11-27T12:26:48,601 DEBUG [StoreCloser-TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/C/6091c17e2d2e4927902eed2303c2887d to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/C/6091c17e2d2e4927902eed2303c2887d 2024-11-27T12:26:48,602 DEBUG [StoreCloser-TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/C/0007d7093f65428e8372f78cd84daf9b to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/C/0007d7093f65428e8372f78cd84daf9b 2024-11-27T12:26:48,602 DEBUG [StoreCloser-TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/C/7dd9abb37bea4c45b38854bae479c2ae to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/C/7dd9abb37bea4c45b38854bae479c2ae 2024-11-27T12:26:48,603 DEBUG [StoreCloser-TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/C/de863092a9e544d1bd0a430d366a1772 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/C/de863092a9e544d1bd0a430d366a1772 2024-11-27T12:26:48,604 DEBUG [StoreCloser-TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/C/f595a0d32eb840ee92a88cb6582e0fbe to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/C/f595a0d32eb840ee92a88cb6582e0fbe 2024-11-27T12:26:48,604 DEBUG [StoreCloser-TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/C/40f8c7350e824ae1b61f49dea7e10abd to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/C/40f8c7350e824ae1b61f49dea7e10abd 2024-11-27T12:26:48,605 DEBUG [StoreCloser-TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/C/2208e1ff36ce449fa6916bcd53534723 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/C/2208e1ff36ce449fa6916bcd53534723 2024-11-27T12:26:48,606 DEBUG [StoreCloser-TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/C/ed88643264c6485db7d05eaa6e27da2b to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/C/ed88643264c6485db7d05eaa6e27da2b 2024-11-27T12:26:48,607 DEBUG [StoreCloser-TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/C/a918db7de0314ec5ac65f60bf2964c84 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/C/a918db7de0314ec5ac65f60bf2964c84 2024-11-27T12:26:48,607 DEBUG [StoreCloser-TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/C/4d0b53bcc56b4f0aadef81262443eaf6 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/C/4d0b53bcc56b4f0aadef81262443eaf6 2024-11-27T12:26:48,608 DEBUG [StoreCloser-TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/C/e482a28d44c842358857d715d860bb9e to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/C/e482a28d44c842358857d715d860bb9e 2024-11-27T12:26:48,608 DEBUG [StoreCloser-TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/C/d2cd42089f9d4b77b846612f55bac727 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/C/d2cd42089f9d4b77b846612f55bac727 2024-11-27T12:26:48,609 DEBUG [StoreCloser-TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/C/2d8d5c8c2d88417f8d9cc618cc56fb41 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/C/2d8d5c8c2d88417f8d9cc618cc56fb41 2024-11-27T12:26:48,610 DEBUG [StoreCloser-TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/C/8bb1bab37c97414bb84d0ffdc4d7c599 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/C/8bb1bab37c97414bb84d0ffdc4d7c599 2024-11-27T12:26:48,610 DEBUG [StoreCloser-TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/C/634736fb747d4b71ae7a14a43ce80fca to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/C/634736fb747d4b71ae7a14a43ce80fca 2024-11-27T12:26:48,614 DEBUG [RS_CLOSE_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/recovered.edits/354.seqid, newMaxSeqId=354, maxSeqId=1 2024-11-27T12:26:48,614 INFO [RS_CLOSE_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315. 2024-11-27T12:26:48,615 DEBUG [RS_CLOSE_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegion(1676): Region close journal for 3144c5c576970df3a2a1c91af06cc315: Waiting for close lock at 1732710407286Running coprocessor pre-close hooks at 1732710407286Disabling compacts and flushes for region at 1732710407286Disabling writes for close at 1732710407286Obtaining lock to block concurrent updates at 1732710407287 (+1 ms)Preparing flush snapshotting stores in 3144c5c576970df3a2a1c91af06cc315 at 1732710407287Finished memstore snapshotting TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=72720, getOffHeapSize=0, getCellsCount=600 at 1732710407287Flushing stores of TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315. at 1732710407289 (+2 ms)Flushing 3144c5c576970df3a2a1c91af06cc315/A: creating writer at 1732710407289Flushing 3144c5c576970df3a2a1c91af06cc315/A: appending metadata at 1732710407296 (+7 ms)Flushing 3144c5c576970df3a2a1c91af06cc315/A: closing flushed file at 1732710407296Flushing 3144c5c576970df3a2a1c91af06cc315/B: creating writer at 1732710407705 (+409 ms)Flushing 3144c5c576970df3a2a1c91af06cc315/B: appending metadata at 1732710407707 (+2 ms)Flushing 3144c5c576970df3a2a1c91af06cc315/B: closing flushed file at 1732710407707Flushing 3144c5c576970df3a2a1c91af06cc315/C: creating writer at 1732710408123 (+416 ms)Flushing 3144c5c576970df3a2a1c91af06cc315/C: appending metadata at 1732710408125 (+2 ms)Flushing 3144c5c576970df3a2a1c91af06cc315/C: closing flushed file at 1732710408125Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6b93afb2: reopening flushed file at 1732710408539 (+414 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@8d2495a: reopening flushed file at 1732710408546 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@62382d1c: reopening flushed file at 1732710408551 (+5 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~71.02 KB/72720, currentSize=0 B/0 for 3144c5c576970df3a2a1c91af06cc315 in 1270ms, sequenceid=351, compaction requested=true at 1732710408556 (+5 ms)Writing region close event to WAL at 1732710408612 (+56 ms)Running coprocessor post-close hooks at 1732710408614 (+2 ms)Closed at 1732710408614 2024-11-27T12:26:48,616 INFO [RS_CLOSE_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] handler.UnassignRegionHandler(157): Closed 3144c5c576970df3a2a1c91af06cc315 2024-11-27T12:26:48,617 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=145 updating hbase:meta row=3144c5c576970df3a2a1c91af06cc315, regionState=CLOSED 2024-11-27T12:26:48,618 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=146, ppid=145, state=RUNNABLE, hasLock=false; CloseRegionProcedure 3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 because future has completed 2024-11-27T12:26:48,620 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=146, resume processing ppid=145 2024-11-27T12:26:48,620 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=146, ppid=145, state=SUCCESS, hasLock=false; CloseRegionProcedure 3144c5c576970df3a2a1c91af06cc315, server=1ba5af43fd99,36251,1732710241631 in 1.4870 sec 2024-11-27T12:26:48,621 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=145, resume processing ppid=144 2024-11-27T12:26:48,621 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=145, ppid=144, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=3144c5c576970df3a2a1c91af06cc315, UNASSIGN in 1.4920 sec 2024-11-27T12:26:48,623 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=144, resume processing ppid=143 2024-11-27T12:26:48,623 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=144, ppid=143, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.4960 sec 2024-11-27T12:26:48,624 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732710408623"}]},"ts":"1732710408623"} 2024-11-27T12:26:48,625 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-27T12:26:48,625 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-27T12:26:48,626 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=143, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.5080 sec 2024-11-27T12:26:49,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-11-27T12:26:49,269 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-27T12:26:49,271 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-11-27T12:26:49,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] procedure2.ProcedureExecutor(1139): Stored pid=147, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-27T12:26:49,301 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=147, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-27T12:26:49,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=147 2024-11-27T12:26:49,303 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=147, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-27T12:26:49,309 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315 2024-11-27T12:26:49,314 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/A, FileablePath, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/B, FileablePath, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/C, FileablePath, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/recovered.edits] 2024-11-27T12:26:49,317 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/A/155f608cb45c435984034fb0d77fc00f to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/A/155f608cb45c435984034fb0d77fc00f 2024-11-27T12:26:49,318 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/A/e61cabf66c19402689c54dc7ad8f2e03 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/A/e61cabf66c19402689c54dc7ad8f2e03 2024-11-27T12:26:49,319 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/A/f02e108fcab140cca46169e85f99c840 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/A/f02e108fcab140cca46169e85f99c840 2024-11-27T12:26:49,320 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/B/50ebca5376a94d04abb07996432641cb to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/B/50ebca5376a94d04abb07996432641cb 2024-11-27T12:26:49,321 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/B/c5d3885675dc45feab0f2e94dd38b96a to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/B/c5d3885675dc45feab0f2e94dd38b96a 2024-11-27T12:26:49,322 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/B/d992f90bba4e4813852be714c476a2c0 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/B/d992f90bba4e4813852be714c476a2c0 2024-11-27T12:26:49,323 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/C/6680f24e696d428cb28ce242d25a2789 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/C/6680f24e696d428cb28ce242d25a2789 2024-11-27T12:26:49,324 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/C/696614b9908b4d918b76e36abb3570fa to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/C/696614b9908b4d918b76e36abb3570fa 2024-11-27T12:26:49,324 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/C/adf12e0eb20141d0be61e61b698127d6 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/C/adf12e0eb20141d0be61e61b698127d6 2024-11-27T12:26:49,326 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/recovered.edits/354.seqid to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315/recovered.edits/354.seqid 2024-11-27T12:26:49,326 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/3144c5c576970df3a2a1c91af06cc315 2024-11-27T12:26:49,326 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-27T12:26:49,328 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=147, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-27T12:26:49,330 WARN [PEWorker-5 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-27T12:26:49,331 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-27T12:26:49,332 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=147, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-27T12:26:49,332 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-27T12:26:49,333 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732710409332"}]},"ts":"9223372036854775807"} 2024-11-27T12:26:49,334 INFO [PEWorker-5 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-27T12:26:49,334 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 3144c5c576970df3a2a1c91af06cc315, NAME => 'TestAcidGuarantees,,1732710375110.3144c5c576970df3a2a1c91af06cc315.', STARTKEY => '', ENDKEY => ''}] 2024-11-27T12:26:49,334 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-27T12:26:49,334 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732710409334"}]},"ts":"9223372036854775807"} 2024-11-27T12:26:49,335 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-27T12:26:49,336 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(133): Finished pid=147, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-27T12:26:49,337 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=147, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 65 msec 2024-11-27T12:26:49,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=147 2024-11-27T12:26:49,409 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-27T12:26:49,412 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-27T12:26:49,422 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testScanAtomicity Thread=237 (was 239), OpenFileDescriptor=441 (was 443), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=372 (was 464), ProcessCount=11 (was 11), AvailableMemoryMB=8650 (was 8669) 2024-11-27T12:26:49,433 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMobGetAtomicity Thread=237, OpenFileDescriptor=441, MaxFileDescriptor=1048576, SystemLoadAverage=372, ProcessCount=11, AvailableMemoryMB=8650 2024-11-27T12:26:49,434 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-27T12:26:49,435 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-27T12:26:49,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] procedure2.ProcedureExecutor(1139): Stored pid=148, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-27T12:26:49,437 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=148, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-27T12:26:49,437 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:26:49,437 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 148 2024-11-27T12:26:49,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-27T12:26:49,438 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=148, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-27T12:26:49,444 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742341_1517 (size=959) 2024-11-27T12:26:49,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-27T12:26:49,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-27T12:26:49,850 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => a2c58e0d0c97538318beef21ebb34095, NAME => 'TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c 2024-11-27T12:26:49,859 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742342_1518 (size=53) 2024-11-27T12:26:50,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-27T12:26:50,261 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T12:26:50,261 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing a2c58e0d0c97538318beef21ebb34095, disabling compactions & flushes 2024-11-27T12:26:50,261 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095. 2024-11-27T12:26:50,262 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095. 2024-11-27T12:26:50,262 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095. after waiting 0 ms 2024-11-27T12:26:50,262 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095. 2024-11-27T12:26:50,262 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095. 2024-11-27T12:26:50,262 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for a2c58e0d0c97538318beef21ebb34095: Waiting for close lock at 1732710410261Disabling compacts and flushes for region at 1732710410261Disabling writes for close at 1732710410262 (+1 ms)Writing region close event to WAL at 1732710410262Closed at 1732710410262 2024-11-27T12:26:50,265 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=148, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-27T12:26:50,265 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732710410265"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732710410265"}]},"ts":"1732710410265"} 2024-11-27T12:26:50,269 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-27T12:26:50,271 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=148, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-27T12:26:50,271 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732710410271"}]},"ts":"1732710410271"} 2024-11-27T12:26:50,274 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-27T12:26:50,274 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=149, ppid=148, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a2c58e0d0c97538318beef21ebb34095, ASSIGN}] 2024-11-27T12:26:50,276 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=149, ppid=148, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a2c58e0d0c97538318beef21ebb34095, ASSIGN 2024-11-27T12:26:50,277 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=149, ppid=148, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=a2c58e0d0c97538318beef21ebb34095, ASSIGN; state=OFFLINE, location=1ba5af43fd99,36251,1732710241631; forceNewPlan=false, retain=false 2024-11-27T12:26:50,428 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=149 updating hbase:meta row=a2c58e0d0c97538318beef21ebb34095, regionState=OPENING, regionLocation=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:50,432 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=149, ppid=148, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=a2c58e0d0c97538318beef21ebb34095, ASSIGN because future has completed 2024-11-27T12:26:50,433 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=150, ppid=149, state=RUNNABLE, hasLock=false; OpenRegionProcedure a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631}] 2024-11-27T12:26:50,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-27T12:26:50,592 INFO [RS_OPEN_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_REGION, pid=150}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095. 2024-11-27T12:26:50,592 DEBUG [RS_OPEN_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_REGION, pid=150}] regionserver.HRegion(7752): Opening region: {ENCODED => a2c58e0d0c97538318beef21ebb34095, NAME => 'TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095.', STARTKEY => '', ENDKEY => ''} 2024-11-27T12:26:50,593 DEBUG [RS_OPEN_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_REGION, pid=150}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees a2c58e0d0c97538318beef21ebb34095 2024-11-27T12:26:50,593 DEBUG [RS_OPEN_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_REGION, pid=150}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T12:26:50,593 DEBUG [RS_OPEN_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_REGION, pid=150}] regionserver.HRegion(7794): checking encryption for a2c58e0d0c97538318beef21ebb34095 2024-11-27T12:26:50,593 DEBUG [RS_OPEN_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_REGION, pid=150}] regionserver.HRegion(7797): checking classloading for a2c58e0d0c97538318beef21ebb34095 2024-11-27T12:26:50,595 INFO [StoreOpener-a2c58e0d0c97538318beef21ebb34095-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region a2c58e0d0c97538318beef21ebb34095 2024-11-27T12:26:50,599 INFO [StoreOpener-a2c58e0d0c97538318beef21ebb34095-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a2c58e0d0c97538318beef21ebb34095 columnFamilyName A 2024-11-27T12:26:50,599 DEBUG [StoreOpener-a2c58e0d0c97538318beef21ebb34095-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:26:50,600 INFO [StoreOpener-a2c58e0d0c97538318beef21ebb34095-1 {}] regionserver.HStore(327): Store=a2c58e0d0c97538318beef21ebb34095/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T12:26:50,600 INFO [StoreOpener-a2c58e0d0c97538318beef21ebb34095-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region a2c58e0d0c97538318beef21ebb34095 2024-11-27T12:26:50,601 INFO [StoreOpener-a2c58e0d0c97538318beef21ebb34095-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a2c58e0d0c97538318beef21ebb34095 columnFamilyName B 2024-11-27T12:26:50,602 DEBUG [StoreOpener-a2c58e0d0c97538318beef21ebb34095-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:26:50,602 INFO [StoreOpener-a2c58e0d0c97538318beef21ebb34095-1 {}] regionserver.HStore(327): Store=a2c58e0d0c97538318beef21ebb34095/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T12:26:50,602 INFO [StoreOpener-a2c58e0d0c97538318beef21ebb34095-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region a2c58e0d0c97538318beef21ebb34095 2024-11-27T12:26:50,603 INFO [StoreOpener-a2c58e0d0c97538318beef21ebb34095-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a2c58e0d0c97538318beef21ebb34095 columnFamilyName C 2024-11-27T12:26:50,603 DEBUG [StoreOpener-a2c58e0d0c97538318beef21ebb34095-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:26:50,604 INFO [StoreOpener-a2c58e0d0c97538318beef21ebb34095-1 {}] regionserver.HStore(327): Store=a2c58e0d0c97538318beef21ebb34095/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T12:26:50,604 DEBUG [RS_OPEN_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_REGION, pid=150}] regionserver.HRegion(1038): replaying wal for a2c58e0d0c97538318beef21ebb34095 2024-11-27T12:26:50,604 DEBUG [RS_OPEN_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_REGION, pid=150}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095 2024-11-27T12:26:50,605 DEBUG [RS_OPEN_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_REGION, pid=150}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095 2024-11-27T12:26:50,606 DEBUG [RS_OPEN_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_REGION, pid=150}] regionserver.HRegion(1048): stopping wal replay for a2c58e0d0c97538318beef21ebb34095 2024-11-27T12:26:50,606 DEBUG [RS_OPEN_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_REGION, pid=150}] regionserver.HRegion(1060): Cleaning up temporary data for a2c58e0d0c97538318beef21ebb34095 2024-11-27T12:26:50,606 DEBUG [RS_OPEN_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_REGION, pid=150}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-27T12:26:50,607 DEBUG [RS_OPEN_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_REGION, pid=150}] regionserver.HRegion(1093): writing seq id for a2c58e0d0c97538318beef21ebb34095 2024-11-27T12:26:50,609 DEBUG [RS_OPEN_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_REGION, pid=150}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-27T12:26:50,609 INFO [RS_OPEN_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_REGION, pid=150}] regionserver.HRegion(1114): Opened a2c58e0d0c97538318beef21ebb34095; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62875217, jitterRate=-0.06308625638484955}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-27T12:26:50,609 DEBUG [RS_OPEN_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_REGION, pid=150}] regionserver.HRegion(1122): Running coprocessor post-open hooks for a2c58e0d0c97538318beef21ebb34095 2024-11-27T12:26:50,610 DEBUG [RS_OPEN_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_REGION, pid=150}] regionserver.HRegion(1006): Region open journal for a2c58e0d0c97538318beef21ebb34095: Running coprocessor pre-open hook at 1732710410593Writing region info on filesystem at 1732710410593Initializing all the Stores at 1732710410595 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732710410595Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732710410595Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732710410595Cleaning up temporary data from old regions at 1732710410606 (+11 ms)Running coprocessor post-open hooks at 1732710410609 (+3 ms)Region opened successfully at 1732710410610 (+1 ms) 2024-11-27T12:26:50,611 INFO [RS_OPEN_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_REGION, pid=150}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., pid=150, masterSystemTime=1732710410587 2024-11-27T12:26:50,612 DEBUG [RS_OPEN_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_REGION, pid=150}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095. 2024-11-27T12:26:50,612 INFO [RS_OPEN_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_REGION, pid=150}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095. 2024-11-27T12:26:50,613 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=149 updating hbase:meta row=a2c58e0d0c97538318beef21ebb34095, regionState=OPEN, openSeqNum=2, regionLocation=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:50,614 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=150, ppid=149, state=RUNNABLE, hasLock=false; OpenRegionProcedure a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 because future has completed 2024-11-27T12:26:50,616 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=150, resume processing ppid=149 2024-11-27T12:26:50,616 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=150, ppid=149, state=SUCCESS, hasLock=false; OpenRegionProcedure a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 in 182 msec 2024-11-27T12:26:50,618 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=149, resume processing ppid=148 2024-11-27T12:26:50,618 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=149, ppid=148, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a2c58e0d0c97538318beef21ebb34095, ASSIGN in 343 msec 2024-11-27T12:26:50,619 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=148, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-27T12:26:50,619 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732710410619"}]},"ts":"1732710410619"} 2024-11-27T12:26:50,620 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-27T12:26:50,621 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=148, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-27T12:26:50,622 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=148, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1860 sec 2024-11-27T12:26:51,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-27T12:26:51,590 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-27T12:26:51,593 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@55741a63, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T12:26:51,593 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1ba5af43fd99,36215,-1 for getting cluster id 2024-11-27T12:26:51,593 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T12:26:51,596 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '200978c2-2c41-4f93-b0b5-127ea1024f37' 2024-11-27T12:26:51,596 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T12:26:51,597 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "200978c2-2c41-4f93-b0b5-127ea1024f37" 2024-11-27T12:26:51,597 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@10416a57, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T12:26:51,597 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1ba5af43fd99,36215,-1] 2024-11-27T12:26:51,598 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T12:26:51,598 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T12:26:51,599 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47638, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T12:26:51,600 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4bef0660, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T12:26:51,600 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T12:26:51,601 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1ba5af43fd99,36251,1732710241631, seqNum=-1] 2024-11-27T12:26:51,601 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T12:26:51,601 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:54756, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T12:26:51,602 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-27T12:26:51,603 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncConnectionImpl(321): The fetched master address is 1ba5af43fd99,36215,1732710240860 2024-11-27T12:26:51,603 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@7ecf00cc 2024-11-27T12:26:51,603 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-27T12:26:51,604 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47654, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-27T12:26:51,605 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-27T12:26:51,605 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.3 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-27T12:26:51,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] procedure2.ProcedureExecutor(1139): Stored pid=151, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-27T12:26:51,616 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742343_1519 (size=995) 2024-11-27T12:26:52,018 DEBUG [PEWorker-4 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.959 2024-11-27T12:26:52,018 INFO [PEWorker-4 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.995 2024-11-27T12:26:52,020 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=152, ppid=151, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-27T12:26:52,022 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=153, ppid=152, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a2c58e0d0c97538318beef21ebb34095, REOPEN/MOVE}] 2024-11-27T12:26:52,023 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=153, ppid=152, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a2c58e0d0c97538318beef21ebb34095, REOPEN/MOVE 2024-11-27T12:26:52,024 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=153 updating hbase:meta row=a2c58e0d0c97538318beef21ebb34095, regionState=CLOSING, regionLocation=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:52,025 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=153, ppid=152, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=a2c58e0d0c97538318beef21ebb34095, REOPEN/MOVE because future has completed 2024-11-27T12:26:52,026 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-27T12:26:52,026 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=154, ppid=153, state=RUNNABLE, hasLock=false; CloseRegionProcedure a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631}] 2024-11-27T12:26:52,180 INFO [RS_CLOSE_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_CLOSE_REGION, pid=154}] handler.UnassignRegionHandler(122): Close a2c58e0d0c97538318beef21ebb34095 2024-11-27T12:26:52,180 DEBUG [RS_CLOSE_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_CLOSE_REGION, pid=154}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-27T12:26:52,180 DEBUG [RS_CLOSE_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_CLOSE_REGION, pid=154}] regionserver.HRegion(1722): Closing a2c58e0d0c97538318beef21ebb34095, disabling compactions & flushes 2024-11-27T12:26:52,180 INFO [RS_CLOSE_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_CLOSE_REGION, pid=154}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095. 2024-11-27T12:26:52,180 DEBUG [RS_CLOSE_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_CLOSE_REGION, pid=154}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095. 2024-11-27T12:26:52,180 DEBUG [RS_CLOSE_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_CLOSE_REGION, pid=154}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095. after waiting 0 ms 2024-11-27T12:26:52,180 DEBUG [RS_CLOSE_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_CLOSE_REGION, pid=154}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095. 2024-11-27T12:26:52,189 DEBUG [RS_CLOSE_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_CLOSE_REGION, pid=154}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-27T12:26:52,191 INFO [RS_CLOSE_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_CLOSE_REGION, pid=154}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095. 2024-11-27T12:26:52,191 DEBUG [RS_CLOSE_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_CLOSE_REGION, pid=154}] regionserver.HRegion(1676): Region close journal for a2c58e0d0c97538318beef21ebb34095: Waiting for close lock at 1732710412180Running coprocessor pre-close hooks at 1732710412180Disabling compacts and flushes for region at 1732710412180Disabling writes for close at 1732710412180Writing region close event to WAL at 1732710412181 (+1 ms)Running coprocessor post-close hooks at 1732710412191 (+10 ms)Closed at 1732710412191 2024-11-27T12:26:52,191 WARN [RS_CLOSE_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_CLOSE_REGION, pid=154}] regionserver.HRegionServer(3299): Not adding moved region record: a2c58e0d0c97538318beef21ebb34095 to self. 2024-11-27T12:26:52,193 INFO [RS_CLOSE_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_CLOSE_REGION, pid=154}] handler.UnassignRegionHandler(157): Closed a2c58e0d0c97538318beef21ebb34095 2024-11-27T12:26:52,194 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=153 updating hbase:meta row=a2c58e0d0c97538318beef21ebb34095, regionState=CLOSED 2024-11-27T12:26:52,195 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=154, ppid=153, state=RUNNABLE, hasLock=false; CloseRegionProcedure a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 because future has completed 2024-11-27T12:26:52,196 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=154, resume processing ppid=153 2024-11-27T12:26:52,196 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=154, ppid=153, state=SUCCESS, hasLock=false; CloseRegionProcedure a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 in 169 msec 2024-11-27T12:26:52,197 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=153, ppid=152, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=a2c58e0d0c97538318beef21ebb34095, REOPEN/MOVE; state=CLOSED, location=1ba5af43fd99,36251,1732710241631; forceNewPlan=false, retain=true 2024-11-27T12:26:52,348 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=153 updating hbase:meta row=a2c58e0d0c97538318beef21ebb34095, regionState=OPENING, regionLocation=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:52,350 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=153, ppid=152, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=a2c58e0d0c97538318beef21ebb34095, REOPEN/MOVE because future has completed 2024-11-27T12:26:52,350 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=155, ppid=153, state=RUNNABLE, hasLock=false; OpenRegionProcedure a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631}] 2024-11-27T12:26:52,508 INFO [RS_OPEN_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_REGION, pid=155}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095. 2024-11-27T12:26:52,509 DEBUG [RS_OPEN_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_REGION, pid=155}] regionserver.HRegion(7752): Opening region: {ENCODED => a2c58e0d0c97538318beef21ebb34095, NAME => 'TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095.', STARTKEY => '', ENDKEY => ''} 2024-11-27T12:26:52,509 DEBUG [RS_OPEN_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_REGION, pid=155}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees a2c58e0d0c97538318beef21ebb34095 2024-11-27T12:26:52,509 DEBUG [RS_OPEN_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_REGION, pid=155}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T12:26:52,510 DEBUG [RS_OPEN_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_REGION, pid=155}] regionserver.HRegion(7794): checking encryption for a2c58e0d0c97538318beef21ebb34095 2024-11-27T12:26:52,510 DEBUG [RS_OPEN_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_REGION, pid=155}] regionserver.HRegion(7797): checking classloading for a2c58e0d0c97538318beef21ebb34095 2024-11-27T12:26:52,513 INFO [StoreOpener-a2c58e0d0c97538318beef21ebb34095-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region a2c58e0d0c97538318beef21ebb34095 2024-11-27T12:26:52,514 INFO [StoreOpener-a2c58e0d0c97538318beef21ebb34095-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a2c58e0d0c97538318beef21ebb34095 columnFamilyName A 2024-11-27T12:26:52,517 DEBUG [StoreOpener-a2c58e0d0c97538318beef21ebb34095-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:26:52,518 INFO [StoreOpener-a2c58e0d0c97538318beef21ebb34095-1 {}] regionserver.HStore(327): Store=a2c58e0d0c97538318beef21ebb34095/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T12:26:52,518 INFO [StoreOpener-a2c58e0d0c97538318beef21ebb34095-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region a2c58e0d0c97538318beef21ebb34095 2024-11-27T12:26:52,519 INFO [StoreOpener-a2c58e0d0c97538318beef21ebb34095-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a2c58e0d0c97538318beef21ebb34095 columnFamilyName B 2024-11-27T12:26:52,519 DEBUG [StoreOpener-a2c58e0d0c97538318beef21ebb34095-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:26:52,520 INFO [StoreOpener-a2c58e0d0c97538318beef21ebb34095-1 {}] regionserver.HStore(327): Store=a2c58e0d0c97538318beef21ebb34095/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T12:26:52,520 INFO [StoreOpener-a2c58e0d0c97538318beef21ebb34095-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region a2c58e0d0c97538318beef21ebb34095 2024-11-27T12:26:52,521 INFO [StoreOpener-a2c58e0d0c97538318beef21ebb34095-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a2c58e0d0c97538318beef21ebb34095 columnFamilyName C 2024-11-27T12:26:52,521 DEBUG [StoreOpener-a2c58e0d0c97538318beef21ebb34095-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:26:52,522 INFO [StoreOpener-a2c58e0d0c97538318beef21ebb34095-1 {}] regionserver.HStore(327): Store=a2c58e0d0c97538318beef21ebb34095/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T12:26:52,522 DEBUG [RS_OPEN_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_REGION, pid=155}] regionserver.HRegion(1038): replaying wal for a2c58e0d0c97538318beef21ebb34095 2024-11-27T12:26:52,523 DEBUG [RS_OPEN_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_REGION, pid=155}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095 2024-11-27T12:26:52,525 DEBUG [RS_OPEN_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_REGION, pid=155}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095 2024-11-27T12:26:52,526 DEBUG [RS_OPEN_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_REGION, pid=155}] regionserver.HRegion(1048): stopping wal replay for a2c58e0d0c97538318beef21ebb34095 2024-11-27T12:26:52,526 DEBUG [RS_OPEN_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_REGION, pid=155}] regionserver.HRegion(1060): Cleaning up temporary data for a2c58e0d0c97538318beef21ebb34095 2024-11-27T12:26:52,527 DEBUG [RS_OPEN_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_REGION, pid=155}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-27T12:26:52,529 DEBUG [RS_OPEN_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_REGION, pid=155}] regionserver.HRegion(1093): writing seq id for a2c58e0d0c97538318beef21ebb34095 2024-11-27T12:26:52,531 INFO [RS_OPEN_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_REGION, pid=155}] regionserver.HRegion(1114): Opened a2c58e0d0c97538318beef21ebb34095; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68567873, jitterRate=0.021740928292274475}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-27T12:26:52,531 DEBUG [RS_OPEN_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_REGION, pid=155}] regionserver.HRegion(1122): Running coprocessor post-open hooks for a2c58e0d0c97538318beef21ebb34095 2024-11-27T12:26:52,532 DEBUG [RS_OPEN_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_REGION, pid=155}] regionserver.HRegion(1006): Region open journal for a2c58e0d0c97538318beef21ebb34095: Running coprocessor pre-open hook at 1732710412510Writing region info on filesystem at 1732710412510Initializing all the Stores at 1732710412512 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732710412512Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732710412512Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732710412512Cleaning up temporary data from old regions at 1732710412526 (+14 ms)Running coprocessor post-open hooks at 1732710412531 (+5 ms)Region opened successfully at 1732710412532 (+1 ms) 2024-11-27T12:26:52,533 INFO [RS_OPEN_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_REGION, pid=155}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., pid=155, masterSystemTime=1732710412503 2024-11-27T12:26:52,535 DEBUG [RS_OPEN_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_REGION, pid=155}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095. 2024-11-27T12:26:52,535 INFO [RS_OPEN_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_OPEN_REGION, pid=155}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095. 2024-11-27T12:26:52,535 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=153 updating hbase:meta row=a2c58e0d0c97538318beef21ebb34095, regionState=OPEN, openSeqNum=5, regionLocation=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:52,537 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=155, ppid=153, state=RUNNABLE, hasLock=false; OpenRegionProcedure a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 because future has completed 2024-11-27T12:26:52,540 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=155, resume processing ppid=153 2024-11-27T12:26:52,540 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=155, ppid=153, state=SUCCESS, hasLock=false; OpenRegionProcedure a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 in 188 msec 2024-11-27T12:26:52,542 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=153, resume processing ppid=152 2024-11-27T12:26:52,542 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=153, ppid=152, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a2c58e0d0c97538318beef21ebb34095, REOPEN/MOVE in 518 msec 2024-11-27T12:26:52,544 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=152, resume processing ppid=151 2024-11-27T12:26:52,544 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=152, ppid=151, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 522 msec 2024-11-27T12:26:52,546 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=151, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 939 msec 2024-11-27T12:26:52,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=151 2024-11-27T12:26:52,546 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-27T12:26:52,547 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4262b017, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T12:26:52,547 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1ba5af43fd99,36215,-1 for getting cluster id 2024-11-27T12:26:52,547 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T12:26:52,548 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '200978c2-2c41-4f93-b0b5-127ea1024f37' 2024-11-27T12:26:52,548 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T12:26:52,548 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "200978c2-2c41-4f93-b0b5-127ea1024f37" 2024-11-27T12:26:52,548 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@498bd267, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T12:26:52,548 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1ba5af43fd99,36215,-1] 2024-11-27T12:26:52,548 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T12:26:52,548 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T12:26:52,549 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58196, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T12:26:52,549 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4ce4e4f1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T12:26:52,550 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4d6b3ea1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T12:26:52,550 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1ba5af43fd99,36215,-1 for getting cluster id 2024-11-27T12:26:52,550 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T12:26:52,550 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '200978c2-2c41-4f93-b0b5-127ea1024f37' 2024-11-27T12:26:52,550 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T12:26:52,550 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "200978c2-2c41-4f93-b0b5-127ea1024f37" 2024-11-27T12:26:52,550 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@31650790, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T12:26:52,550 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1ba5af43fd99,36215,-1] 2024-11-27T12:26:52,551 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T12:26:52,551 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T12:26:52,551 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58218, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T12:26:52,551 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@73ee262a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T12:26:52,552 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@263b0516, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T12:26:52,552 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1ba5af43fd99,36215,-1 for getting cluster id 2024-11-27T12:26:52,552 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T12:26:52,552 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '200978c2-2c41-4f93-b0b5-127ea1024f37' 2024-11-27T12:26:52,552 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T12:26:52,552 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "200978c2-2c41-4f93-b0b5-127ea1024f37" 2024-11-27T12:26:52,553 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4c6e2cd8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T12:26:52,553 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1ba5af43fd99,36215,-1] 2024-11-27T12:26:52,553 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T12:26:52,553 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T12:26:52,553 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58226, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T12:26:52,554 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5cb2244c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T12:26:52,554 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@792e8aea, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T12:26:52,554 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1ba5af43fd99,36215,-1 for getting cluster id 2024-11-27T12:26:52,554 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T12:26:52,555 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '200978c2-2c41-4f93-b0b5-127ea1024f37' 2024-11-27T12:26:52,555 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T12:26:52,555 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "200978c2-2c41-4f93-b0b5-127ea1024f37" 2024-11-27T12:26:52,555 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19b2e28c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T12:26:52,555 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1ba5af43fd99,36215,-1] 2024-11-27T12:26:52,555 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T12:26:52,555 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T12:26:52,556 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58244, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T12:26:52,556 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f50d3cc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T12:26:52,557 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@43322a56, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T12:26:52,557 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1ba5af43fd99,36215,-1 for getting cluster id 2024-11-27T12:26:52,557 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T12:26:52,558 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '200978c2-2c41-4f93-b0b5-127ea1024f37' 2024-11-27T12:26:52,558 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T12:26:52,558 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "200978c2-2c41-4f93-b0b5-127ea1024f37" 2024-11-27T12:26:52,558 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@51fc08d8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T12:26:52,558 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1ba5af43fd99,36215,-1] 2024-11-27T12:26:52,558 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T12:26:52,558 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T12:26:52,559 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58268, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T12:26:52,560 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1d5ab7f0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T12:26:52,560 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2e638e47, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T12:26:52,560 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1ba5af43fd99,36215,-1 for getting cluster id 2024-11-27T12:26:52,560 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T12:26:52,561 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '200978c2-2c41-4f93-b0b5-127ea1024f37' 2024-11-27T12:26:52,561 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T12:26:52,561 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "200978c2-2c41-4f93-b0b5-127ea1024f37" 2024-11-27T12:26:52,561 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ff8f751, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T12:26:52,561 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1ba5af43fd99,36215,-1] 2024-11-27T12:26:52,561 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T12:26:52,562 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T12:26:52,562 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58296, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T12:26:52,563 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4442af52, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T12:26:52,563 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@42aa7a6a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T12:26:52,563 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1ba5af43fd99,36215,-1 for getting cluster id 2024-11-27T12:26:52,563 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T12:26:52,564 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '200978c2-2c41-4f93-b0b5-127ea1024f37' 2024-11-27T12:26:52,564 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T12:26:52,564 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "200978c2-2c41-4f93-b0b5-127ea1024f37" 2024-11-27T12:26:52,564 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1e346c5b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T12:26:52,564 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1ba5af43fd99,36215,-1] 2024-11-27T12:26:52,565 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T12:26:52,565 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T12:26:52,565 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58320, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T12:26:52,566 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@33543a6a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T12:26:52,566 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f9cbeba, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T12:26:52,566 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1ba5af43fd99,36215,-1 for getting cluster id 2024-11-27T12:26:52,566 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T12:26:52,567 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '200978c2-2c41-4f93-b0b5-127ea1024f37' 2024-11-27T12:26:52,567 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T12:26:52,567 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "200978c2-2c41-4f93-b0b5-127ea1024f37" 2024-11-27T12:26:52,567 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@603bed20, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T12:26:52,568 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1ba5af43fd99,36215,-1] 2024-11-27T12:26:52,568 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T12:26:52,568 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T12:26:52,568 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58336, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T12:26:52,569 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2a6e18e2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T12:26:52,570 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3012edbb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T12:26:52,570 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1ba5af43fd99,36215,-1 for getting cluster id 2024-11-27T12:26:52,570 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T12:26:52,570 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '200978c2-2c41-4f93-b0b5-127ea1024f37' 2024-11-27T12:26:52,570 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T12:26:52,571 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "200978c2-2c41-4f93-b0b5-127ea1024f37" 2024-11-27T12:26:52,571 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@65068e9b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T12:26:52,571 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1ba5af43fd99,36215,-1] 2024-11-27T12:26:52,571 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T12:26:52,571 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T12:26:52,572 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58356, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T12:26:52,572 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4ddb44d0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T12:26:52,573 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6e3f87a6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T12:26:52,573 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1ba5af43fd99,36215,-1 for getting cluster id 2024-11-27T12:26:52,573 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T12:26:52,573 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '200978c2-2c41-4f93-b0b5-127ea1024f37' 2024-11-27T12:26:52,573 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T12:26:52,573 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "200978c2-2c41-4f93-b0b5-127ea1024f37" 2024-11-27T12:26:52,573 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7baacefe, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T12:26:52,573 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1ba5af43fd99,36215,-1] 2024-11-27T12:26:52,574 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T12:26:52,574 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T12:26:52,574 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58364, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T12:26:52,575 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@77eb831a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T12:26:52,575 DEBUG [Thread-2345 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T12:26:52,575 DEBUG [Thread-2336 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T12:26:52,575 DEBUG [Thread-2332 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T12:26:52,575 DEBUG [Thread-2347 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T12:26:52,575 DEBUG [Thread-2334 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T12:26:52,576 DEBUG [Thread-2343 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T12:26:52,576 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-27T12:26:52,576 DEBUG [Thread-2328 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T12:26:52,576 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1ba5af43fd99,36251,1732710241631, seqNum=-1] 2024-11-27T12:26:52,576 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T12:26:52,576 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1ba5af43fd99,36251,1732710241631, seqNum=-1] 2024-11-27T12:26:52,576 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1ba5af43fd99,36251,1732710241631, seqNum=-1] 2024-11-27T12:26:52,576 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1ba5af43fd99,36251,1732710241631, seqNum=-1] 2024-11-27T12:26:52,576 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T12:26:52,576 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T12:26:52,576 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1ba5af43fd99,36251,1732710241631, seqNum=-1] 2024-11-27T12:26:52,576 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T12:26:52,576 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T12:26:52,577 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1ba5af43fd99,36251,1732710241631, seqNum=-1] 2024-11-27T12:26:52,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] procedure2.ProcedureExecutor(1139): Stored pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees 2024-11-27T12:26:52,577 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1ba5af43fd99,36251,1732710241631, seqNum=-1] 2024-11-27T12:26:52,577 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T12:26:52,577 DEBUG [Thread-2330 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T12:26:52,577 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T12:26:52,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-27T12:26:52,577 DEBUG [Thread-2339 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T12:26:52,578 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:35478, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T12:26:52,578 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T12:26:52,578 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:35484, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T12:26:52,578 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:35492, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T12:26:52,578 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:35496, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T12:26:52,578 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1ba5af43fd99,36251,1732710241631, seqNum=-1] 2024-11-27T12:26:52,578 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:35454, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T12:26:52,578 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:35442, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T12:26:52,578 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:35468, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T12:26:52,578 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T12:26:52,578 DEBUG [Thread-2341 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T12:26:52,579 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T12:26:52,579 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=157, ppid=156, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T12:26:52,579 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1ba5af43fd99,36251,1732710241631, seqNum=-1] 2024-11-27T12:26:52,579 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:35506, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T12:26:52,579 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1ba5af43fd99,36251,1732710241631, seqNum=-1] 2024-11-27T12:26:52,579 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T12:26:52,579 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T12:26:52,580 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:35514, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T12:26:52,580 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:35516, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T12:26:52,580 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5] 2024-11-27T12:26:52,580 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5] 2024-11-27T12:26:52,581 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5] 2024-11-27T12:26:52,582 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5] 2024-11-27T12:26:52,582 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5] 2024-11-27T12:26:52,583 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5] 2024-11-27T12:26:52,583 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5] 2024-11-27T12:26:52,584 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5] 2024-11-27T12:26:52,584 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5] 2024-11-27T12:26:52,584 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5] 2024-11-27T12:26:52,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(8855): Flush requested on a2c58e0d0c97538318beef21ebb34095 2024-11-27T12:26:52,588 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T12:26:52,588 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a2c58e0d0c97538318beef21ebb34095 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-27T12:26:52,609 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411272893a15ee2a8493da2b6dd136649e7a4_a2c58e0d0c97538318beef21ebb34095 is 50, key is test_row_0/A:col10/1732710412587/Put/seqid=0 2024-11-27T12:26:52,612 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742344_1520 (size=14594) 2024-11-27T12:26:52,638 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:52,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35506 deadline: 1732710472634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:52,638 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:52,638 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:52,639 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:26:52,639 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:52,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35442 deadline: 1732710472635, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:52,639 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:52,639 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:52,639 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:52,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35496 deadline: 1732710472636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:52,639 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:26:52,639 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:52,639 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:52,639 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:52,639 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:26:52,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35468 deadline: 1732710472636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:52,639 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:52,640 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:52,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35492 deadline: 1732710472636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:52,640 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:52,640 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:26:52,640 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:52,640 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:52,640 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:26:52,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-27T12:26:52,731 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-27T12:26:52,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095. 2024-11-27T12:26:52,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095. as already flushing 2024-11-27T12:26:52,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095. 2024-11-27T12:26:52,732 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => a2c58e0d0c97538318beef21ebb34095, NAME => 'TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:26:52,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => a2c58e0d0c97538318beef21ebb34095, NAME => 'TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:26:52,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36215 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a2c58e0d0c97538318beef21ebb34095, NAME => 'TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a2c58e0d0c97538318beef21ebb34095, NAME => 'TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:26:52,750 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:52,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35506 deadline: 1732710472748, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:52,750 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:52,750 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:52,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35442 deadline: 1732710472748, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:52,750 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:52,750 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:26:52,750 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:52,750 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:52,750 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:52,750 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:26:52,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35496 deadline: 1732710472748, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:52,751 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:52,751 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:52,751 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:52,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35468 deadline: 1732710472749, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:52,751 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:26:52,751 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:52,751 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:52,751 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:52,751 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:26:52,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35492 deadline: 1732710472749, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:52,751 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:52,751 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:52,751 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:26:52,884 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-27T12:26:52,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095. 2024-11-27T12:26:52,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095. as already flushing 2024-11-27T12:26:52,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095. 2024-11-27T12:26:52,884 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => a2c58e0d0c97538318beef21ebb34095, NAME => 'TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:26:52,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => a2c58e0d0c97538318beef21ebb34095, NAME => 'TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:26:52,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36215 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a2c58e0d0c97538318beef21ebb34095, NAME => 'TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a2c58e0d0c97538318beef21ebb34095, NAME => 'TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:26:52,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-27T12:26:52,959 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:52,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35506 deadline: 1732710472957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:52,960 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:52,960 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:52,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35442 deadline: 1732710472958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:52,960 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:52,960 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:26:52,960 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:52,960 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:52,960 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:52,960 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:26:52,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35468 deadline: 1732710472958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:52,960 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:52,960 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:52,960 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:52,960 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:26:52,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35496 deadline: 1732710472958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:52,961 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:52,960 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:52,961 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:52,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35492 deadline: 1732710472958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:52,961 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:26:52,961 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:52,961 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:52,961 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:26:53,013 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:26:53,016 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411272893a15ee2a8493da2b6dd136649e7a4_a2c58e0d0c97538318beef21ebb34095 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411272893a15ee2a8493da2b6dd136649e7a4_a2c58e0d0c97538318beef21ebb34095 2024-11-27T12:26:53,016 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/.tmp/A/11a73d08d3b543efbb7bb2183d051530, store: [table=TestAcidGuarantees family=A region=a2c58e0d0c97538318beef21ebb34095] 2024-11-27T12:26:53,017 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/.tmp/A/11a73d08d3b543efbb7bb2183d051530 is 175, key is test_row_0/A:col10/1732710412587/Put/seqid=0 2024-11-27T12:26:53,020 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742345_1521 (size=39549) 2024-11-27T12:26:53,036 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-27T12:26:53,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095. 2024-11-27T12:26:53,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095. as already flushing 2024-11-27T12:26:53,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095. 2024-11-27T12:26:53,037 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => a2c58e0d0c97538318beef21ebb34095, NAME => 'TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:26:53,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => a2c58e0d0c97538318beef21ebb34095, NAME => 'TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:26:53,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36215 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a2c58e0d0c97538318beef21ebb34095, NAME => 'TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a2c58e0d0c97538318beef21ebb34095, NAME => 'TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:26:53,189 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-27T12:26:53,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095. 2024-11-27T12:26:53,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095. as already flushing 2024-11-27T12:26:53,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095. 2024-11-27T12:26:53,189 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => a2c58e0d0c97538318beef21ebb34095, NAME => 'TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:26:53,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => a2c58e0d0c97538318beef21ebb34095, NAME => 'TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:26:53,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36215 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a2c58e0d0c97538318beef21ebb34095, NAME => 'TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a2c58e0d0c97538318beef21ebb34095, NAME => 'TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:26:53,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-27T12:26:53,269 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:53,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35506 deadline: 1732710473268, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:53,270 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:53,270 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:53,270 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:53,270 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:26:53,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35442 deadline: 1732710473268, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:53,270 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:53,270 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:53,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35468 deadline: 1732710473268, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:53,270 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:53,270 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:26:53,270 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:53,270 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:53,270 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:53,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35492 deadline: 1732710473269, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:53,270 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:26:53,270 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:53,270 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:53,270 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:53,270 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:26:53,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35496 deadline: 1732710473269, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:53,271 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:53,271 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:53,271 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:26:53,341 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-27T12:26:53,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095. 2024-11-27T12:26:53,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095. as already flushing 2024-11-27T12:26:53,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095. 2024-11-27T12:26:53,342 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => a2c58e0d0c97538318beef21ebb34095, NAME => 'TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:26:53,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => a2c58e0d0c97538318beef21ebb34095, NAME => 'TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:26:53,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36215 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a2c58e0d0c97538318beef21ebb34095, NAME => 'TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a2c58e0d0c97538318beef21ebb34095, NAME => 'TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:26:53,421 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=17, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/.tmp/A/11a73d08d3b543efbb7bb2183d051530 2024-11-27T12:26:53,442 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/.tmp/B/c74cd445ae004974b3accff81f8d434f is 50, key is test_row_0/B:col10/1732710412587/Put/seqid=0 2024-11-27T12:26:53,445 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742346_1522 (size=12001) 2024-11-27T12:26:53,445 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/.tmp/B/c74cd445ae004974b3accff81f8d434f 2024-11-27T12:26:53,468 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/.tmp/C/16dc89550181433ca2d05b2cfe013ebc is 50, key is test_row_0/C:col10/1732710412587/Put/seqid=0 2024-11-27T12:26:53,471 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742347_1523 (size=12001) 2024-11-27T12:26:53,494 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-27T12:26:53,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095. 2024-11-27T12:26:53,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095. as already flushing 2024-11-27T12:26:53,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095. 2024-11-27T12:26:53,495 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => a2c58e0d0c97538318beef21ebb34095, NAME => 'TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:26:53,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => a2c58e0d0c97538318beef21ebb34095, NAME => 'TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:26:53,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36215 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a2c58e0d0c97538318beef21ebb34095, NAME => 'TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a2c58e0d0c97538318beef21ebb34095, NAME => 'TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:26:53,647 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-27T12:26:53,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095. 2024-11-27T12:26:53,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095. as already flushing 2024-11-27T12:26:53,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095. 2024-11-27T12:26:53,647 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => a2c58e0d0c97538318beef21ebb34095, NAME => 'TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:26:53,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => a2c58e0d0c97538318beef21ebb34095, NAME => 'TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:26:53,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36215 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a2c58e0d0c97538318beef21ebb34095, NAME => 'TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a2c58e0d0c97538318beef21ebb34095, NAME => 'TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:26:53,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-27T12:26:53,779 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:53,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35506 deadline: 1732710473777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:53,779 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:53,779 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:53,779 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:53,779 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:26:53,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35442 deadline: 1732710473777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:53,780 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:53,780 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:53,780 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:53,780 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:26:53,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35468 deadline: 1732710473778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:53,780 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:53,780 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:53,780 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:26:53,780 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:53,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35492 deadline: 1732710473778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:53,780 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:53,780 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:53,780 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:53,780 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:26:53,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35496 deadline: 1732710473778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:53,780 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:53,781 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:53,781 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:26:53,799 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-27T12:26:53,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095. 2024-11-27T12:26:53,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095. as already flushing 2024-11-27T12:26:53,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095. 2024-11-27T12:26:53,800 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => a2c58e0d0c97538318beef21ebb34095, NAME => 'TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:26:53,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => a2c58e0d0c97538318beef21ebb34095, NAME => 'TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:26:53,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36215 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a2c58e0d0c97538318beef21ebb34095, NAME => 'TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a2c58e0d0c97538318beef21ebb34095, NAME => 'TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:26:53,871 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/.tmp/C/16dc89550181433ca2d05b2cfe013ebc 2024-11-27T12:26:53,875 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/.tmp/A/11a73d08d3b543efbb7bb2183d051530 as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/A/11a73d08d3b543efbb7bb2183d051530 2024-11-27T12:26:53,878 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/A/11a73d08d3b543efbb7bb2183d051530, entries=200, sequenceid=17, filesize=38.6 K 2024-11-27T12:26:53,878 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/.tmp/B/c74cd445ae004974b3accff81f8d434f as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/B/c74cd445ae004974b3accff81f8d434f 2024-11-27T12:26:53,881 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/B/c74cd445ae004974b3accff81f8d434f, entries=150, sequenceid=17, filesize=11.7 K 2024-11-27T12:26:53,881 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/.tmp/C/16dc89550181433ca2d05b2cfe013ebc as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/C/16dc89550181433ca2d05b2cfe013ebc 2024-11-27T12:26:53,884 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/C/16dc89550181433ca2d05b2cfe013ebc, entries=150, sequenceid=17, filesize=11.7 K 2024-11-27T12:26:53,885 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=140.89 KB/144270 for a2c58e0d0c97538318beef21ebb34095 in 1297ms, sequenceid=17, compaction requested=false 2024-11-27T12:26:53,885 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a2c58e0d0c97538318beef21ebb34095: 2024-11-27T12:26:53,952 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-27T12:26:53,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095. 2024-11-27T12:26:53,952 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2902): Flushing a2c58e0d0c97538318beef21ebb34095 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-27T12:26:53,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127972984d15b714cf58a203ea1b5f1ecfe_a2c58e0d0c97538318beef21ebb34095 is 50, key is test_row_0/A:col10/1732710412628/Put/seqid=0 2024-11-27T12:26:53,961 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742348_1524 (size=12154) 2024-11-27T12:26:54,134 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-27T12:26:54,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:26:54,407 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127972984d15b714cf58a203ea1b5f1ecfe_a2c58e0d0c97538318beef21ebb34095 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127972984d15b714cf58a203ea1b5f1ecfe_a2c58e0d0c97538318beef21ebb34095 2024-11-27T12:26:54,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/.tmp/A/f3fca4d358d1462ca7244c4e1d2d482e, store: [table=TestAcidGuarantees family=A region=a2c58e0d0c97538318beef21ebb34095] 2024-11-27T12:26:54,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/.tmp/A/f3fca4d358d1462ca7244c4e1d2d482e is 175, key is test_row_0/A:col10/1732710412628/Put/seqid=0 2024-11-27T12:26:54,411 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742349_1525 (size=30955) 2024-11-27T12:26:54,412 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=41, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/.tmp/A/f3fca4d358d1462ca7244c4e1d2d482e 2024-11-27T12:26:54,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/.tmp/B/0fc4365b36d64a0fb7c899060949712a is 50, key is test_row_0/B:col10/1732710412628/Put/seqid=0 2024-11-27T12:26:54,428 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742350_1526 (size=12001) 2024-11-27T12:26:54,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-27T12:26:54,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(8855): Flush requested on a2c58e0d0c97538318beef21ebb34095 2024-11-27T12:26:54,791 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095. as already flushing 2024-11-27T12:26:54,798 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:54,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35442 deadline: 1732710474795, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:54,799 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:54,799 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:54,799 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:54,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35492 deadline: 1732710474796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:54,799 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:26:54,799 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:54,799 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:54,799 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:54,799 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:26:54,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35496 deadline: 1732710474796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:54,799 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:54,799 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:54,799 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:54,799 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:26:54,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35506 deadline: 1732710474797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:54,799 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:54,799 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:54,799 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:26:54,799 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:54,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35468 deadline: 1732710474798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:54,800 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 1ba5af43fd99,36251,1732710241631 for 'test_row_2' in a2c58e0d0c97538318beef21ebb34095 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2164 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T12:26:54,800 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:54,800 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:54,800 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:26:54,829 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=41 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/.tmp/B/0fc4365b36d64a0fb7c899060949712a 2024-11-27T12:26:54,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/.tmp/C/7f4cf58d304f4f888e3306fc9a2648f1 is 50, key is test_row_0/C:col10/1732710412628/Put/seqid=0 2024-11-27T12:26:54,853 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742351_1527 (size=12001) 2024-11-27T12:26:54,909 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:54,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35442 deadline: 1732710474908, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:54,909 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:54,910 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:54,910 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:26:54,909 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:54,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35492 deadline: 1732710474908, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:54,910 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:54,910 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:54,910 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:26:54,910 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:54,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35496 deadline: 1732710474908, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:54,910 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:54,910 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:54,910 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:26:54,910 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:54,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35506 deadline: 1732710474908, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:54,911 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:54,911 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:54,911 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:26:55,119 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:55,119 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:55,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35442 deadline: 1732710475117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:55,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35492 deadline: 1732710475118, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:55,119 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:55,119 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:55,120 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:55,120 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:55,120 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:55,120 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:55,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35496 deadline: 1732710475118, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:55,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35506 deadline: 1732710475118, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:55,120 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:26:55,120 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:26:55,120 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:55,120 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:55,120 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:55,120 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:55,120 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:26:55,120 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:26:55,253 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=41 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/.tmp/C/7f4cf58d304f4f888e3306fc9a2648f1 2024-11-27T12:26:55,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/.tmp/A/f3fca4d358d1462ca7244c4e1d2d482e as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/A/f3fca4d358d1462ca7244c4e1d2d482e 2024-11-27T12:26:55,262 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/A/f3fca4d358d1462ca7244c4e1d2d482e, entries=150, sequenceid=41, filesize=30.2 K 2024-11-27T12:26:55,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/.tmp/B/0fc4365b36d64a0fb7c899060949712a as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/B/0fc4365b36d64a0fb7c899060949712a 2024-11-27T12:26:55,267 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/B/0fc4365b36d64a0fb7c899060949712a, entries=150, sequenceid=41, filesize=11.7 K 2024-11-27T12:26:55,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/.tmp/C/7f4cf58d304f4f888e3306fc9a2648f1 as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/C/7f4cf58d304f4f888e3306fc9a2648f1 2024-11-27T12:26:55,271 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/C/7f4cf58d304f4f888e3306fc9a2648f1, entries=150, sequenceid=41, filesize=11.7 K 2024-11-27T12:26:55,272 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for a2c58e0d0c97538318beef21ebb34095 in 1320ms, sequenceid=41, compaction requested=false 2024-11-27T12:26:55,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2603): Flush status journal for a2c58e0d0c97538318beef21ebb34095: 2024-11-27T12:26:55,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095. 2024-11-27T12:26:55,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=157 2024-11-27T12:26:55,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36215 {}] master.HMaster(4169): Remote procedure done, pid=157 2024-11-27T12:26:55,275 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=157, resume processing ppid=156 2024-11-27T12:26:55,275 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=157, ppid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6950 sec 2024-11-27T12:26:55,277 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees in 2.7000 sec 2024-11-27T12:26:55,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(8855): Flush requested on a2c58e0d0c97538318beef21ebb34095 2024-11-27T12:26:55,430 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T12:26:55,430 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a2c58e0d0c97538318beef21ebb34095 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-27T12:26:55,436 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127d0bafcb00d7c476aac2827178eaddcf1_a2c58e0d0c97538318beef21ebb34095 is 50, key is test_row_0/A:col10/1732710415430/Put/seqid=0 2024-11-27T12:26:55,440 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742352_1528 (size=12154) 2024-11-27T12:26:55,441 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:26:55,443 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127d0bafcb00d7c476aac2827178eaddcf1_a2c58e0d0c97538318beef21ebb34095 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127d0bafcb00d7c476aac2827178eaddcf1_a2c58e0d0c97538318beef21ebb34095 2024-11-27T12:26:55,444 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/.tmp/A/f7abff7285e24386ab04f8d15be53802, store: [table=TestAcidGuarantees family=A region=a2c58e0d0c97538318beef21ebb34095] 2024-11-27T12:26:55,444 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/.tmp/A/f7abff7285e24386ab04f8d15be53802 is 175, key is test_row_0/A:col10/1732710415430/Put/seqid=0 2024-11-27T12:26:55,447 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742353_1529 (size=30955) 2024-11-27T12:26:55,456 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:55,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35506 deadline: 1732710475451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:55,456 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:55,456 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:55,456 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:26:55,456 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:55,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35496 deadline: 1732710475452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:55,456 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:55,456 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:55,456 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:26:55,457 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:55,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35492 deadline: 1732710475453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:55,457 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:55,457 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:55,457 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:26:55,457 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:55,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35442 deadline: 1732710475454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:55,457 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:55,458 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:55,458 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:26:55,558 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:55,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35506 deadline: 1732710475557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:55,559 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:55,559 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:55,559 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:26:55,570 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:55,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35496 deadline: 1732710475568, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:55,571 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:55,571 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:55,571 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:55,571 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:26:55,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35492 deadline: 1732710475568, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:55,571 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:55,571 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:55,571 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:26:55,571 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:55,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35442 deadline: 1732710475568, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:55,571 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:55,571 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:55,571 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:26:55,768 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:55,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35506 deadline: 1732710475767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:55,768 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:55,768 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:55,768 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:26:55,781 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:55,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35496 deadline: 1732710475778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:55,781 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:55,781 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:55,781 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:55,782 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:26:55,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35442 deadline: 1732710475778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:55,782 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:55,782 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:55,782 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:26:55,782 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:55,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35492 deadline: 1732710475778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:55,782 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:55,782 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:55,782 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:26:55,848 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=57, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/.tmp/A/f7abff7285e24386ab04f8d15be53802 2024-11-27T12:26:55,854 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/.tmp/B/76bbaef2dc4e4276b7a161b3768bb0aa is 50, key is test_row_0/B:col10/1732710415430/Put/seqid=0 2024-11-27T12:26:55,856 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742354_1530 (size=12001) 2024-11-27T12:26:56,080 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:56,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35506 deadline: 1732710476078, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:56,081 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:56,081 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:56,081 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:26:56,091 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:56,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35496 deadline: 1732710476088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:56,091 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:56,091 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:56,091 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:56,091 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:26:56,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35442 deadline: 1732710476088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:56,091 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:56,091 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:56,091 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:56,091 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:26:56,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35492 deadline: 1732710476088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:56,091 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:56,091 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:56,092 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:26:56,257 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=57 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/.tmp/B/76bbaef2dc4e4276b7a161b3768bb0aa 2024-11-27T12:26:56,263 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/.tmp/C/2a2b141351ca4b2a85a2509b2be79e24 is 50, key is test_row_0/C:col10/1732710415430/Put/seqid=0 2024-11-27T12:26:56,266 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742355_1531 (size=12001) 2024-11-27T12:26:56,266 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=57 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/.tmp/C/2a2b141351ca4b2a85a2509b2be79e24 2024-11-27T12:26:56,269 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/.tmp/A/f7abff7285e24386ab04f8d15be53802 as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/A/f7abff7285e24386ab04f8d15be53802 2024-11-27T12:26:56,272 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/A/f7abff7285e24386ab04f8d15be53802, entries=150, sequenceid=57, filesize=30.2 K 2024-11-27T12:26:56,272 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/.tmp/B/76bbaef2dc4e4276b7a161b3768bb0aa as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/B/76bbaef2dc4e4276b7a161b3768bb0aa 2024-11-27T12:26:56,275 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/B/76bbaef2dc4e4276b7a161b3768bb0aa, entries=150, sequenceid=57, filesize=11.7 K 2024-11-27T12:26:56,275 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/.tmp/C/2a2b141351ca4b2a85a2509b2be79e24 as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/C/2a2b141351ca4b2a85a2509b2be79e24 2024-11-27T12:26:56,278 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/C/2a2b141351ca4b2a85a2509b2be79e24, entries=150, sequenceid=57, filesize=11.7 K 2024-11-27T12:26:56,278 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=114.05 KB/116790 for a2c58e0d0c97538318beef21ebb34095 in 848ms, sequenceid=57, compaction requested=true 2024-11-27T12:26:56,278 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a2c58e0d0c97538318beef21ebb34095: 2024-11-27T12:26:56,278 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a2c58e0d0c97538318beef21ebb34095:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T12:26:56,279 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T12:26:56,279 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T12:26:56,279 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a2c58e0d0c97538318beef21ebb34095:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T12:26:56,279 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T12:26:56,279 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T12:26:56,279 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a2c58e0d0c97538318beef21ebb34095:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T12:26:56,279 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T12:26:56,279 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T12:26:56,279 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101459 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T12:26:56,279 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HStore(1541): a2c58e0d0c97538318beef21ebb34095/A is initiating minor compaction (all files) 2024-11-27T12:26:56,279 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HStore(1541): a2c58e0d0c97538318beef21ebb34095/B is initiating minor compaction (all files) 2024-11-27T12:26:56,279 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a2c58e0d0c97538318beef21ebb34095/A in TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095. 2024-11-27T12:26:56,279 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a2c58e0d0c97538318beef21ebb34095/B in TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095. 2024-11-27T12:26:56,279 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/B/c74cd445ae004974b3accff81f8d434f, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/B/0fc4365b36d64a0fb7c899060949712a, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/B/76bbaef2dc4e4276b7a161b3768bb0aa] into tmpdir=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/.tmp, totalSize=35.2 K 2024-11-27T12:26:56,279 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/A/11a73d08d3b543efbb7bb2183d051530, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/A/f3fca4d358d1462ca7244c4e1d2d482e, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/A/f7abff7285e24386ab04f8d15be53802] into tmpdir=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/.tmp, totalSize=99.1 K 2024-11-27T12:26:56,279 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=53.85 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095. 2024-11-27T12:26:56,280 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095. files: [hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/A/11a73d08d3b543efbb7bb2183d051530, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/A/f3fca4d358d1462ca7244c4e1d2d482e, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/A/f7abff7285e24386ab04f8d15be53802] 2024-11-27T12:26:56,280 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.Compactor(225): Compacting c74cd445ae004974b3accff81f8d434f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=17, earliestPutTs=1732710412586 2024-11-27T12:26:56,280 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] compactions.Compactor(225): Compacting 11a73d08d3b543efbb7bb2183d051530, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=17, earliestPutTs=1732710412584 2024-11-27T12:26:56,280 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.Compactor(225): Compacting 0fc4365b36d64a0fb7c899060949712a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=41, earliestPutTs=1732710412628 2024-11-27T12:26:56,280 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.Compactor(225): Compacting 76bbaef2dc4e4276b7a161b3768bb0aa, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=57, earliestPutTs=1732710414796 2024-11-27T12:26:56,281 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] compactions.Compactor(225): Compacting f3fca4d358d1462ca7244c4e1d2d482e, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=41, earliestPutTs=1732710412628 2024-11-27T12:26:56,281 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] compactions.Compactor(225): Compacting f7abff7285e24386ab04f8d15be53802, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=57, earliestPutTs=1732710414796 2024-11-27T12:26:56,294 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=a2c58e0d0c97538318beef21ebb34095] 2024-11-27T12:26:56,295 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241127aad60940fa994f3fa07414c4adc6e124_a2c58e0d0c97538318beef21ebb34095 store=[table=TestAcidGuarantees family=A region=a2c58e0d0c97538318beef21ebb34095] 2024-11-27T12:26:56,297 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a2c58e0d0c97538318beef21ebb34095#B#compaction#447 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 53.85 MB/second 2024-11-27T12:26:56,297 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241127aad60940fa994f3fa07414c4adc6e124_a2c58e0d0c97538318beef21ebb34095, store=[table=TestAcidGuarantees family=A region=a2c58e0d0c97538318beef21ebb34095] 2024-11-27T12:26:56,297 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127aad60940fa994f3fa07414c4adc6e124_a2c58e0d0c97538318beef21ebb34095 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=a2c58e0d0c97538318beef21ebb34095] 2024-11-27T12:26:56,297 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/.tmp/B/7e84549f5d284ac5b86eaee3bc3d57ed is 50, key is test_row_0/B:col10/1732710415430/Put/seqid=0 2024-11-27T12:26:56,309 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742356_1532 (size=4469) 2024-11-27T12:26:56,313 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742357_1533 (size=12104) 2024-11-27T12:26:56,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(8855): Flush requested on a2c58e0d0c97538318beef21ebb34095 2024-11-27T12:26:56,590 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T12:26:56,590 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a2c58e0d0c97538318beef21ebb34095 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-27T12:26:56,596 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127668429895811467f8c9bb86d93a6eb11_a2c58e0d0c97538318beef21ebb34095 is 50, key is test_row_0/A:col10/1732710415453/Put/seqid=0 2024-11-27T12:26:56,599 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742358_1534 (size=14594) 2024-11-27T12:26:56,612 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:56,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35442 deadline: 1732710476608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:56,613 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:56,613 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:56,613 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:26:56,613 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:56,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35506 deadline: 1732710476610, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:56,613 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:56,613 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:56,613 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:26:56,616 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:56,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35496 deadline: 1732710476612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:56,617 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:56,617 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:56,617 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:26:56,617 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:56,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35492 deadline: 1732710476613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:56,617 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:56,617 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:56,617 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:26:56,710 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a2c58e0d0c97538318beef21ebb34095#A#compaction#446 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-27T12:26:56,711 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/.tmp/A/b5469766cfcb45a7bee390bfa229c9bc is 175, key is test_row_0/A:col10/1732710415430/Put/seqid=0 2024-11-27T12:26:56,714 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742359_1535 (size=31058) 2024-11-27T12:26:56,719 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/.tmp/B/7e84549f5d284ac5b86eaee3bc3d57ed as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/B/7e84549f5d284ac5b86eaee3bc3d57ed 2024-11-27T12:26:56,720 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:56,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35442 deadline: 1732710476717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:56,720 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:56,720 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:56,720 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:56,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35506 deadline: 1732710476718, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:56,720 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:26:56,721 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:56,721 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:56,721 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:26:56,722 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a2c58e0d0c97538318beef21ebb34095/B of a2c58e0d0c97538318beef21ebb34095 into 7e84549f5d284ac5b86eaee3bc3d57ed(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T12:26:56,723 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a2c58e0d0c97538318beef21ebb34095: 2024-11-27T12:26:56,723 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., storeName=a2c58e0d0c97538318beef21ebb34095/B, priority=13, startTime=1732710416279; duration=0sec 2024-11-27T12:26:56,723 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T12:26:56,723 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a2c58e0d0c97538318beef21ebb34095:B 2024-11-27T12:26:56,723 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T12:26:56,723 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T12:26:56,723 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HStore(1541): a2c58e0d0c97538318beef21ebb34095/C is initiating minor compaction (all files) 2024-11-27T12:26:56,723 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a2c58e0d0c97538318beef21ebb34095/C in TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095. 2024-11-27T12:26:56,724 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/C/16dc89550181433ca2d05b2cfe013ebc, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/C/7f4cf58d304f4f888e3306fc9a2648f1, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/C/2a2b141351ca4b2a85a2509b2be79e24] into tmpdir=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/.tmp, totalSize=35.2 K 2024-11-27T12:26:56,724 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.Compactor(225): Compacting 16dc89550181433ca2d05b2cfe013ebc, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=17, earliestPutTs=1732710412586 2024-11-27T12:26:56,724 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.Compactor(225): Compacting 7f4cf58d304f4f888e3306fc9a2648f1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=41, earliestPutTs=1732710412628 2024-11-27T12:26:56,724 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.Compactor(225): Compacting 2a2b141351ca4b2a85a2509b2be79e24, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=57, earliestPutTs=1732710414796 2024-11-27T12:26:56,730 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a2c58e0d0c97538318beef21ebb34095#C#compaction#449 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-27T12:26:56,730 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/.tmp/C/979cc434a6d64166a2b6fb9296f915b5 is 50, key is test_row_0/C:col10/1732710415430/Put/seqid=0 2024-11-27T12:26:56,730 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:56,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35496 deadline: 1732710476728, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:56,731 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:56,731 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:56,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35492 deadline: 1732710476728, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:56,731 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:56,731 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:26:56,731 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:56,731 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:56,731 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:26:56,733 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742360_1536 (size=12104) 2024-11-27T12:26:56,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-27T12:26:56,759 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T12:26:56,759 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36215 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-27T12:26:56,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36215 {}] procedure2.ProcedureExecutor(1139): Stored pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees 2024-11-27T12:26:56,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-27T12:26:56,760 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T12:26:56,761 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=158, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T12:26:56,761 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=159, ppid=158, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T12:26:56,820 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:56,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35468 deadline: 1732710476818, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:56,820 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 1ba5af43fd99,36251,1732710241631 for 'test_row_2' in a2c58e0d0c97538318beef21ebb34095 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4184 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T12:26:56,821 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:56,821 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:56,821 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:26:56,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-27T12:26:56,913 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-27T12:26:56,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095. 2024-11-27T12:26:56,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095. as already flushing 2024-11-27T12:26:56,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095. 2024-11-27T12:26:56,913 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => a2c58e0d0c97538318beef21ebb34095, NAME => 'TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:26:56,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => a2c58e0d0c97538318beef21ebb34095, NAME => 'TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:26:56,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36215 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a2c58e0d0c97538318beef21ebb34095, NAME => 'TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a2c58e0d0c97538318beef21ebb34095, NAME => 'TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:26:56,930 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:56,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35442 deadline: 1732710476928, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:56,930 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:56,930 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:56,930 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:56,930 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:26:56,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35506 deadline: 1732710476928, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:56,931 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:56,931 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:56,931 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:26:56,942 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:56,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35496 deadline: 1732710476938, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:56,943 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:56,943 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:56,943 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:26:56,943 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:56,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35492 deadline: 1732710476938, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:56,943 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:56,943 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:56,943 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:26:57,000 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:26:57,003 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127668429895811467f8c9bb86d93a6eb11_a2c58e0d0c97538318beef21ebb34095 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127668429895811467f8c9bb86d93a6eb11_a2c58e0d0c97538318beef21ebb34095 2024-11-27T12:26:57,004 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/.tmp/A/51773740d62b4903bd86cd2de3b40fce, store: [table=TestAcidGuarantees family=A region=a2c58e0d0c97538318beef21ebb34095] 2024-11-27T12:26:57,004 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/.tmp/A/51773740d62b4903bd86cd2de3b40fce is 175, key is test_row_0/A:col10/1732710415453/Put/seqid=0 2024-11-27T12:26:57,007 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742361_1537 (size=39549) 2024-11-27T12:26:57,066 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-27T12:26:57,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095. 2024-11-27T12:26:57,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095. as already flushing 2024-11-27T12:26:57,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095. 2024-11-27T12:26:57,066 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => a2c58e0d0c97538318beef21ebb34095, NAME => 'TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:26:57,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => a2c58e0d0c97538318beef21ebb34095, NAME => 'TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:26:57,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36215 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a2c58e0d0c97538318beef21ebb34095, NAME => 'TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a2c58e0d0c97538318beef21ebb34095, NAME => 'TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:26:57,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-27T12:26:57,118 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/.tmp/A/b5469766cfcb45a7bee390bfa229c9bc as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/A/b5469766cfcb45a7bee390bfa229c9bc 2024-11-27T12:26:57,121 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a2c58e0d0c97538318beef21ebb34095/A of a2c58e0d0c97538318beef21ebb34095 into b5469766cfcb45a7bee390bfa229c9bc(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T12:26:57,121 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a2c58e0d0c97538318beef21ebb34095: 2024-11-27T12:26:57,121 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., storeName=a2c58e0d0c97538318beef21ebb34095/A, priority=13, startTime=1732710416278; duration=0sec 2024-11-27T12:26:57,121 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T12:26:57,121 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a2c58e0d0c97538318beef21ebb34095:A 2024-11-27T12:26:57,137 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/.tmp/C/979cc434a6d64166a2b6fb9296f915b5 as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/C/979cc434a6d64166a2b6fb9296f915b5 2024-11-27T12:26:57,141 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a2c58e0d0c97538318beef21ebb34095/C of a2c58e0d0c97538318beef21ebb34095 into 979cc434a6d64166a2b6fb9296f915b5(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T12:26:57,141 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a2c58e0d0c97538318beef21ebb34095: 2024-11-27T12:26:57,141 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., storeName=a2c58e0d0c97538318beef21ebb34095/C, priority=13, startTime=1732710416279; duration=0sec 2024-11-27T12:26:57,141 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T12:26:57,141 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a2c58e0d0c97538318beef21ebb34095:C 2024-11-27T12:26:57,218 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-27T12:26:57,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095. 2024-11-27T12:26:57,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095. as already flushing 2024-11-27T12:26:57,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095. 2024-11-27T12:26:57,219 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => a2c58e0d0c97538318beef21ebb34095, NAME => 'TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:26:57,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => a2c58e0d0c97538318beef21ebb34095, NAME => 'TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:26:57,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36215 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a2c58e0d0c97538318beef21ebb34095, NAME => 'TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a2c58e0d0c97538318beef21ebb34095, NAME => 'TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:26:57,239 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:57,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35442 deadline: 1732710477237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:57,239 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:57,239 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:57,239 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:57,239 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:26:57,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35506 deadline: 1732710477237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:57,240 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:57,240 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:57,240 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:26:57,250 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:57,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35496 deadline: 1732710477248, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:57,250 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:57,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35492 deadline: 1732710477248, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:57,250 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:57,250 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:57,250 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:26:57,251 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:57,251 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:57,251 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:26:57,371 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-27T12:26:57,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095. 2024-11-27T12:26:57,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095. as already flushing 2024-11-27T12:26:57,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095. 2024-11-27T12:26:57,371 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => a2c58e0d0c97538318beef21ebb34095, NAME => 'TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:26:57,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => a2c58e0d0c97538318beef21ebb34095, NAME => 'TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:26:57,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36215 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a2c58e0d0c97538318beef21ebb34095, NAME => 'TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a2c58e0d0c97538318beef21ebb34095, NAME => 'TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:26:57,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-27T12:26:57,408 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=78, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/.tmp/A/51773740d62b4903bd86cd2de3b40fce 2024-11-27T12:26:57,414 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/.tmp/B/37ecd6560e8c4f4fbae5f12cc4a23042 is 50, key is test_row_0/B:col10/1732710415453/Put/seqid=0 2024-11-27T12:26:57,417 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742362_1538 (size=12001) 2024-11-27T12:26:57,523 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-27T12:26:57,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095. 2024-11-27T12:26:57,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095. as already flushing 2024-11-27T12:26:57,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095. 2024-11-27T12:26:57,524 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => a2c58e0d0c97538318beef21ebb34095, NAME => 'TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:26:57,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => a2c58e0d0c97538318beef21ebb34095, NAME => 'TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:26:57,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36215 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a2c58e0d0c97538318beef21ebb34095, NAME => 'TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a2c58e0d0c97538318beef21ebb34095, NAME => 'TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:26:57,676 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-27T12:26:57,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095. 2024-11-27T12:26:57,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095. as already flushing 2024-11-27T12:26:57,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095. 2024-11-27T12:26:57,676 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => a2c58e0d0c97538318beef21ebb34095, NAME => 'TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:26:57,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => a2c58e0d0c97538318beef21ebb34095, NAME => 'TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:26:57,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36215 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a2c58e0d0c97538318beef21ebb34095, NAME => 'TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a2c58e0d0c97538318beef21ebb34095, NAME => 'TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:26:57,749 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:57,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35442 deadline: 1732710477748, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:57,749 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:57,749 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:57,749 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:57,749 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:26:57,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35506 deadline: 1732710477748, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:57,750 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:57,750 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:57,750 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:26:57,760 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:57,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35496 deadline: 1732710477758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:57,760 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:57,760 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:57,760 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:57,760 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:26:57,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35492 deadline: 1732710477758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:57,761 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:57,761 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:57,761 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:26:57,818 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=78 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/.tmp/B/37ecd6560e8c4f4fbae5f12cc4a23042 2024-11-27T12:26:57,823 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/.tmp/C/05ab749ddd5940548076ec08fbf75af8 is 50, key is test_row_0/C:col10/1732710415453/Put/seqid=0 2024-11-27T12:26:57,829 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-27T12:26:57,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095. 2024-11-27T12:26:57,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095. as already flushing 2024-11-27T12:26:57,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095. 2024-11-27T12:26:57,829 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => a2c58e0d0c97538318beef21ebb34095, NAME => 'TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:26:57,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => a2c58e0d0c97538318beef21ebb34095, NAME => 'TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:26:57,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36215 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a2c58e0d0c97538318beef21ebb34095, NAME => 'TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a2c58e0d0c97538318beef21ebb34095, NAME => 'TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:26:57,830 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742363_1539 (size=12001) 2024-11-27T12:26:57,831 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=78 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/.tmp/C/05ab749ddd5940548076ec08fbf75af8 2024-11-27T12:26:57,834 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/.tmp/A/51773740d62b4903bd86cd2de3b40fce as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/A/51773740d62b4903bd86cd2de3b40fce 2024-11-27T12:26:57,837 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/A/51773740d62b4903bd86cd2de3b40fce, entries=200, sequenceid=78, filesize=38.6 K 2024-11-27T12:26:57,838 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/.tmp/B/37ecd6560e8c4f4fbae5f12cc4a23042 as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/B/37ecd6560e8c4f4fbae5f12cc4a23042 2024-11-27T12:26:57,840 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/B/37ecd6560e8c4f4fbae5f12cc4a23042, entries=150, sequenceid=78, filesize=11.7 K 2024-11-27T12:26:57,841 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/.tmp/C/05ab749ddd5940548076ec08fbf75af8 as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/C/05ab749ddd5940548076ec08fbf75af8 2024-11-27T12:26:57,844 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/C/05ab749ddd5940548076ec08fbf75af8, entries=150, sequenceid=78, filesize=11.7 K 2024-11-27T12:26:57,844 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=87.22 KB/89310 for a2c58e0d0c97538318beef21ebb34095 in 1254ms, sequenceid=78, compaction requested=false 2024-11-27T12:26:57,845 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a2c58e0d0c97538318beef21ebb34095: 2024-11-27T12:26:57,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-27T12:26:57,982 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-27T12:26:57,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095. 2024-11-27T12:26:57,982 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2902): Flushing a2c58e0d0c97538318beef21ebb34095 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-27T12:26:57,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112795ab0450871342d0b7077743b306df6a_a2c58e0d0c97538318beef21ebb34095 is 50, key is test_row_0/A:col10/1732710416611/Put/seqid=0 2024-11-27T12:26:57,991 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742364_1540 (size=12154) 2024-11-27T12:26:58,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:26:58,394 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112795ab0450871342d0b7077743b306df6a_a2c58e0d0c97538318beef21ebb34095 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112795ab0450871342d0b7077743b306df6a_a2c58e0d0c97538318beef21ebb34095 2024-11-27T12:26:58,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/.tmp/A/50ff434510a643108f15bdce75ac1a92, store: [table=TestAcidGuarantees family=A region=a2c58e0d0c97538318beef21ebb34095] 2024-11-27T12:26:58,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/.tmp/A/50ff434510a643108f15bdce75ac1a92 is 175, key is test_row_0/A:col10/1732710416611/Put/seqid=0 2024-11-27T12:26:58,398 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742365_1541 (size=30955) 2024-11-27T12:26:58,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(8855): Flush requested on a2c58e0d0c97538318beef21ebb34095 2024-11-27T12:26:58,760 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095. as already flushing 2024-11-27T12:26:58,790 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:58,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35496 deadline: 1732710478784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:58,791 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:58,791 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:58,791 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:26:58,794 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:58,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35492 deadline: 1732710478788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:58,794 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:58,794 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:58,794 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:26:58,795 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:58,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35506 deadline: 1732710478790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:58,795 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:58,795 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:58,795 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:58,795 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:26:58,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35442 deadline: 1732710478790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:58,795 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:58,796 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:58,796 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:26:58,799 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=97, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/.tmp/A/50ff434510a643108f15bdce75ac1a92 2024-11-27T12:26:58,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/.tmp/B/15a1546f025a48faa931bc98bb25fc15 is 50, key is test_row_0/B:col10/1732710416611/Put/seqid=0 2024-11-27T12:26:58,807 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742366_1542 (size=12001) 2024-11-27T12:26:58,899 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:58,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35496 deadline: 1732710478898, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:58,900 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:58,900 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:58,900 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:58,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35506 deadline: 1732710478898, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:58,900 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:26:58,900 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:58,900 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:58,900 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:58,900 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:26:58,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35492 deadline: 1732710478898, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:58,900 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:58,900 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:58,900 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:58,900 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:26:58,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35442 deadline: 1732710478898, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:58,900 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:58,901 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:58,901 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:26:58,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-27T12:26:59,109 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:59,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35496 deadline: 1732710479108, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:59,110 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:59,110 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:59,110 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:59,110 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:26:59,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35492 deadline: 1732710479108, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:59,110 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:59,110 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:59,110 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:59,110 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:26:59,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35506 deadline: 1732710479108, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:59,110 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:59,110 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:59,110 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:26:59,110 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:59,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35442 deadline: 1732710479108, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:59,111 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:59,111 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:59,111 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:26:59,208 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=97 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/.tmp/B/15a1546f025a48faa931bc98bb25fc15 2024-11-27T12:26:59,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/.tmp/C/b2c41d0741fd48ce85f1ddf020eed9fd is 50, key is test_row_0/C:col10/1732710416611/Put/seqid=0 2024-11-27T12:26:59,220 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742367_1543 (size=12001) 2024-11-27T12:26:59,221 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=97 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/.tmp/C/b2c41d0741fd48ce85f1ddf020eed9fd 2024-11-27T12:26:59,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/.tmp/A/50ff434510a643108f15bdce75ac1a92 as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/A/50ff434510a643108f15bdce75ac1a92 2024-11-27T12:26:59,228 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/A/50ff434510a643108f15bdce75ac1a92, entries=150, sequenceid=97, filesize=30.2 K 2024-11-27T12:26:59,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/.tmp/B/15a1546f025a48faa931bc98bb25fc15 as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/B/15a1546f025a48faa931bc98bb25fc15 2024-11-27T12:26:59,232 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/B/15a1546f025a48faa931bc98bb25fc15, entries=150, sequenceid=97, filesize=11.7 K 2024-11-27T12:26:59,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/.tmp/C/b2c41d0741fd48ce85f1ddf020eed9fd as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/C/b2c41d0741fd48ce85f1ddf020eed9fd 2024-11-27T12:26:59,235 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/C/b2c41d0741fd48ce85f1ddf020eed9fd, entries=150, sequenceid=97, filesize=11.7 K 2024-11-27T12:26:59,236 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=114.05 KB/116790 for a2c58e0d0c97538318beef21ebb34095 in 1254ms, sequenceid=97, compaction requested=true 2024-11-27T12:26:59,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2603): Flush status journal for a2c58e0d0c97538318beef21ebb34095: 2024-11-27T12:26:59,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095. 2024-11-27T12:26:59,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=159 2024-11-27T12:26:59,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36215 {}] master.HMaster(4169): Remote procedure done, pid=159 2024-11-27T12:26:59,239 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=159, resume processing ppid=158 2024-11-27T12:26:59,239 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=159, ppid=158, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.4760 sec 2024-11-27T12:26:59,241 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=158, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees in 2.4800 sec 2024-11-27T12:26:59,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(8855): Flush requested on a2c58e0d0c97538318beef21ebb34095 2024-11-27T12:26:59,469 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T12:26:59,469 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a2c58e0d0c97538318beef21ebb34095 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-27T12:26:59,476 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127b5e9cf79e3614698808f7b7f9dbf3d7b_a2c58e0d0c97538318beef21ebb34095 is 50, key is test_row_0/A:col10/1732710419420/Put/seqid=0 2024-11-27T12:26:59,481 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742368_1544 (size=12154) 2024-11-27T12:26:59,481 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:26:59,484 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127b5e9cf79e3614698808f7b7f9dbf3d7b_a2c58e0d0c97538318beef21ebb34095 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127b5e9cf79e3614698808f7b7f9dbf3d7b_a2c58e0d0c97538318beef21ebb34095 2024-11-27T12:26:59,485 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/.tmp/A/f5313edae9e74de994a3b363bd20a7e9, store: [table=TestAcidGuarantees family=A region=a2c58e0d0c97538318beef21ebb34095] 2024-11-27T12:26:59,485 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/.tmp/A/f5313edae9e74de994a3b363bd20a7e9 is 175, key is test_row_0/A:col10/1732710419420/Put/seqid=0 2024-11-27T12:26:59,488 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:59,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35506 deadline: 1732710479484, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:59,489 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:59,489 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:59,489 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:59,489 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:26:59,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35496 deadline: 1732710479485, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:59,489 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742369_1545 (size=30955) 2024-11-27T12:26:59,489 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:59,489 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:59,489 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:26:59,489 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:59,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35442 deadline: 1732710479485, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:59,490 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:59,490 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:59,490 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:26:59,493 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:59,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35492 deadline: 1732710479489, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:59,493 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:59,493 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:59,494 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:26:59,600 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:59,600 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:59,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35496 deadline: 1732710479597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:59,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35442 deadline: 1732710479598, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:59,600 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:59,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35506 deadline: 1732710479598, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:59,600 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:59,600 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:59,600 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:59,600 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:59,600 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:59,600 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:26:59,600 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:26:59,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35492 deadline: 1732710479598, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:59,600 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:59,600 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:59,600 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:59,600 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:26:59,600 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:59,600 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:26:59,810 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:59,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35496 deadline: 1732710479808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:59,811 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:59,811 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:59,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35442 deadline: 1732710479808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:59,811 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:59,811 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:59,811 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:26:59,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35492 deadline: 1732710479808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:59,811 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:59,811 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:59,811 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:26:59,811 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:26:59,811 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:59,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35506 deadline: 1732710479808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:26:59,811 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:59,811 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:26:59,811 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:59,811 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:26:59,811 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:26:59,889 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=120, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/.tmp/A/f5313edae9e74de994a3b363bd20a7e9 2024-11-27T12:26:59,895 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/.tmp/B/80cde32d8e3042b7994566a0370754ea is 50, key is test_row_0/B:col10/1732710419420/Put/seqid=0 2024-11-27T12:26:59,907 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742370_1546 (size=12001) 2024-11-27T12:27:00,015 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-27T12:27:00,120 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:27:00,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35496 deadline: 1732710480118, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:27:00,120 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:27:00,120 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:27:00,120 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:27:00,120 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:27:00,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35442 deadline: 1732710480118, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:27:00,121 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:27:00,121 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:27:00,121 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:27:00,121 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:27:00,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35492 deadline: 1732710480118, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:27:00,121 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:27:00,121 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:27:00,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35506 deadline: 1732710480118, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:27:00,121 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:27:00,121 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:27:00,121 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:27:00,121 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:27:00,121 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:27:00,307 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=120 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/.tmp/B/80cde32d8e3042b7994566a0370754ea 2024-11-27T12:27:00,313 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/.tmp/C/a1d88299e8e94c7d94c6349417cc0e14 is 50, key is test_row_0/C:col10/1732710419420/Put/seqid=0 2024-11-27T12:27:00,329 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742371_1547 (size=12001) 2024-11-27T12:27:00,330 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=120 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/.tmp/C/a1d88299e8e94c7d94c6349417cc0e14 2024-11-27T12:27:00,333 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/.tmp/A/f5313edae9e74de994a3b363bd20a7e9 as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/A/f5313edae9e74de994a3b363bd20a7e9 2024-11-27T12:27:00,336 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/A/f5313edae9e74de994a3b363bd20a7e9, entries=150, sequenceid=120, filesize=30.2 K 2024-11-27T12:27:00,337 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/.tmp/B/80cde32d8e3042b7994566a0370754ea as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/B/80cde32d8e3042b7994566a0370754ea 2024-11-27T12:27:00,340 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/B/80cde32d8e3042b7994566a0370754ea, entries=150, sequenceid=120, filesize=11.7 K 2024-11-27T12:27:00,340 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/.tmp/C/a1d88299e8e94c7d94c6349417cc0e14 as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/C/a1d88299e8e94c7d94c6349417cc0e14 2024-11-27T12:27:00,343 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/C/a1d88299e8e94c7d94c6349417cc0e14, entries=150, sequenceid=120, filesize=11.7 K 2024-11-27T12:27:00,343 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for a2c58e0d0c97538318beef21ebb34095 in 874ms, sequenceid=120, compaction requested=true 2024-11-27T12:27:00,343 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a2c58e0d0c97538318beef21ebb34095: 2024-11-27T12:27:00,344 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a2c58e0d0c97538318beef21ebb34095:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T12:27:00,344 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-27T12:27:00,344 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T12:27:00,344 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a2c58e0d0c97538318beef21ebb34095:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T12:27:00,344 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T12:27:00,344 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a2c58e0d0c97538318beef21ebb34095:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T12:27:00,344 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-27T12:27:00,344 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T12:27:00,344 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 132517 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-27T12:27:00,344 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-27T12:27:00,344 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HStore(1541): a2c58e0d0c97538318beef21ebb34095/B is initiating minor compaction (all files) 2024-11-27T12:27:00,344 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HStore(1541): a2c58e0d0c97538318beef21ebb34095/A is initiating minor compaction (all files) 2024-11-27T12:27:00,344 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a2c58e0d0c97538318beef21ebb34095/B in TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095. 2024-11-27T12:27:00,344 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a2c58e0d0c97538318beef21ebb34095/A in TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095. 2024-11-27T12:27:00,345 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/A/b5469766cfcb45a7bee390bfa229c9bc, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/A/51773740d62b4903bd86cd2de3b40fce, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/A/50ff434510a643108f15bdce75ac1a92, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/A/f5313edae9e74de994a3b363bd20a7e9] into tmpdir=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/.tmp, totalSize=129.4 K 2024-11-27T12:27:00,345 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/B/7e84549f5d284ac5b86eaee3bc3d57ed, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/B/37ecd6560e8c4f4fbae5f12cc4a23042, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/B/15a1546f025a48faa931bc98bb25fc15, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/B/80cde32d8e3042b7994566a0370754ea] into tmpdir=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/.tmp, totalSize=47.0 K 2024-11-27T12:27:00,345 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=53.85 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095. 2024-11-27T12:27:00,345 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095. files: [hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/A/b5469766cfcb45a7bee390bfa229c9bc, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/A/51773740d62b4903bd86cd2de3b40fce, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/A/50ff434510a643108f15bdce75ac1a92, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/A/f5313edae9e74de994a3b363bd20a7e9] 2024-11-27T12:27:00,346 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] compactions.Compactor(225): Compacting b5469766cfcb45a7bee390bfa229c9bc, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=57, earliestPutTs=1732710414796 2024-11-27T12:27:00,346 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.Compactor(225): Compacting 7e84549f5d284ac5b86eaee3bc3d57ed, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=57, earliestPutTs=1732710414796 2024-11-27T12:27:00,346 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] compactions.Compactor(225): Compacting 51773740d62b4903bd86cd2de3b40fce, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1732710415451 2024-11-27T12:27:00,346 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.Compactor(225): Compacting 37ecd6560e8c4f4fbae5f12cc4a23042, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1732710415452 2024-11-27T12:27:00,347 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] compactions.Compactor(225): Compacting 50ff434510a643108f15bdce75ac1a92, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1732710416608 2024-11-27T12:27:00,347 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.Compactor(225): Compacting 15a1546f025a48faa931bc98bb25fc15, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1732710416608 2024-11-27T12:27:00,347 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] compactions.Compactor(225): Compacting f5313edae9e74de994a3b363bd20a7e9, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1732710419419 2024-11-27T12:27:00,347 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.Compactor(225): Compacting 80cde32d8e3042b7994566a0370754ea, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1732710419419 2024-11-27T12:27:00,352 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=a2c58e0d0c97538318beef21ebb34095] 2024-11-27T12:27:00,354 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241127842fb7df4536449aaf85cc90269d8b36_a2c58e0d0c97538318beef21ebb34095 store=[table=TestAcidGuarantees family=A region=a2c58e0d0c97538318beef21ebb34095] 2024-11-27T12:27:00,356 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a2c58e0d0c97538318beef21ebb34095#B#compaction#459 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 53.85 MB/second 2024-11-27T12:27:00,356 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/.tmp/B/416010e191074cfd839210937a8bae9e is 50, key is test_row_0/B:col10/1732710419420/Put/seqid=0 2024-11-27T12:27:00,356 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241127842fb7df4536449aaf85cc90269d8b36_a2c58e0d0c97538318beef21ebb34095, store=[table=TestAcidGuarantees family=A region=a2c58e0d0c97538318beef21ebb34095] 2024-11-27T12:27:00,357 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127842fb7df4536449aaf85cc90269d8b36_a2c58e0d0c97538318beef21ebb34095 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=a2c58e0d0c97538318beef21ebb34095] 2024-11-27T12:27:00,362 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742372_1548 (size=12241) 2024-11-27T12:27:00,365 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/.tmp/B/416010e191074cfd839210937a8bae9e as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/B/416010e191074cfd839210937a8bae9e 2024-11-27T12:27:00,369 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in a2c58e0d0c97538318beef21ebb34095/B of a2c58e0d0c97538318beef21ebb34095 into 416010e191074cfd839210937a8bae9e(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T12:27:00,370 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a2c58e0d0c97538318beef21ebb34095: 2024-11-27T12:27:00,370 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., storeName=a2c58e0d0c97538318beef21ebb34095/B, priority=12, startTime=1732710420344; duration=0sec 2024-11-27T12:27:00,370 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T12:27:00,370 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a2c58e0d0c97538318beef21ebb34095:B 2024-11-27T12:27:00,370 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-27T12:27:00,371 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-27T12:27:00,371 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HStore(1541): a2c58e0d0c97538318beef21ebb34095/C is initiating minor compaction (all files) 2024-11-27T12:27:00,371 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a2c58e0d0c97538318beef21ebb34095/C in TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095. 2024-11-27T12:27:00,371 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/C/979cc434a6d64166a2b6fb9296f915b5, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/C/05ab749ddd5940548076ec08fbf75af8, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/C/b2c41d0741fd48ce85f1ddf020eed9fd, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/C/a1d88299e8e94c7d94c6349417cc0e14] into tmpdir=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/.tmp, totalSize=47.0 K 2024-11-27T12:27:00,371 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.Compactor(225): Compacting 979cc434a6d64166a2b6fb9296f915b5, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=57, earliestPutTs=1732710414796 2024-11-27T12:27:00,371 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.Compactor(225): Compacting 05ab749ddd5940548076ec08fbf75af8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1732710415452 2024-11-27T12:27:00,372 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.Compactor(225): Compacting b2c41d0741fd48ce85f1ddf020eed9fd, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1732710416608 2024-11-27T12:27:00,372 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.Compactor(225): Compacting a1d88299e8e94c7d94c6349417cc0e14, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1732710419419 2024-11-27T12:27:00,377 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742373_1549 (size=4469) 2024-11-27T12:27:00,378 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a2c58e0d0c97538318beef21ebb34095#C#compaction#460 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 53.85 MB/second 2024-11-27T12:27:00,379 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/.tmp/C/7fa7e87809084578bb0b283ab73c8d84 is 50, key is test_row_0/C:col10/1732710419420/Put/seqid=0 2024-11-27T12:27:00,390 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742374_1550 (size=12241) 2024-11-27T12:27:00,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(8855): Flush requested on a2c58e0d0c97538318beef21ebb34095 2024-11-27T12:27:00,630 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T12:27:00,630 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a2c58e0d0c97538318beef21ebb34095 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-27T12:27:00,636 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127e3d48b15db7d4e008af80505ae19ebe6_a2c58e0d0c97538318beef21ebb34095 is 50, key is test_row_0/A:col10/1732710419483/Put/seqid=0 2024-11-27T12:27:00,639 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742375_1551 (size=12304) 2024-11-27T12:27:00,683 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:27:00,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35496 deadline: 1732710480679, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:27:00,683 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:27:00,683 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:27:00,683 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:27:00,686 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:27:00,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35442 deadline: 1732710480681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:27:00,687 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:27:00,687 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:27:00,687 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:27:00,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35492 deadline: 1732710480681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:27:00,687 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:27:00,687 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:27:00,687 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:27:00,687 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:27:00,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35506 deadline: 1732710480682, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:27:00,687 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:27:00,687 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:27:00,687 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:27:00,687 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:27:00,779 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a2c58e0d0c97538318beef21ebb34095#A#compaction#458 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-27T12:27:00,779 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/.tmp/A/0663abe7b9644947ba30247d80ae337d is 175, key is test_row_0/A:col10/1732710419420/Put/seqid=0 2024-11-27T12:27:00,782 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742376_1552 (size=31195) 2024-11-27T12:27:00,791 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:27:00,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35496 deadline: 1732710480788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:27:00,791 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:27:00,792 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:27:00,792 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:27:00,794 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/.tmp/C/7fa7e87809084578bb0b283ab73c8d84 as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/C/7fa7e87809084578bb0b283ab73c8d84 2024-11-27T12:27:00,798 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in a2c58e0d0c97538318beef21ebb34095/C of a2c58e0d0c97538318beef21ebb34095 into 7fa7e87809084578bb0b283ab73c8d84(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T12:27:00,798 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a2c58e0d0c97538318beef21ebb34095: 2024-11-27T12:27:00,798 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., storeName=a2c58e0d0c97538318beef21ebb34095/C, priority=12, startTime=1732710420344; duration=0sec 2024-11-27T12:27:00,798 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T12:27:00,798 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a2c58e0d0c97538318beef21ebb34095:C 2024-11-27T12:27:00,799 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:27:00,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35442 deadline: 1732710480797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:27:00,799 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:27:00,799 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:27:00,799 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:27:00,799 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:27:00,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35506 deadline: 1732710480798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:27:00,799 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:27:00,799 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:27:00,799 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:27:00,799 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:27:00,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35492 deadline: 1732710480798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:27:00,800 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:27:00,800 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:27:00,800 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:27:00,841 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:27:00,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35468 deadline: 1732710480838, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:27:00,841 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 1ba5af43fd99,36251,1732710241631 for 'test_row_2' in a2c58e0d0c97538318beef21ebb34095 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8205 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T12:27:00,842 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:27:00,842 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:27:00,842 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:27:00,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-27T12:27:00,928 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T12:27:00,929 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36215 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-27T12:27:00,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36215 {}] procedure2.ProcedureExecutor(1139): Stored pid=160, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees 2024-11-27T12:27:00,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-27T12:27:00,930 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=160, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T12:27:00,931 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=160, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T12:27:00,931 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=161, ppid=160, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T12:27:00,999 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:27:00,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35496 deadline: 1732710480998, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:27:00,999 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:27:00,999 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:27:00,999 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:27:01,009 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:27:01,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35442 deadline: 1732710481008, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:27:01,009 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:27:01,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35506 deadline: 1732710481008, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:27:01,009 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:27:01,009 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:27:01,009 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:27:01,009 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:27:01,009 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:27:01,009 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:27:01,009 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:27:01,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35492 deadline: 1732710481008, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:27:01,010 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:27:01,010 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:27:01,010 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:27:01,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-27T12:27:01,039 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:27:01,042 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127e3d48b15db7d4e008af80505ae19ebe6_a2c58e0d0c97538318beef21ebb34095 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127e3d48b15db7d4e008af80505ae19ebe6_a2c58e0d0c97538318beef21ebb34095 2024-11-27T12:27:01,042 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/.tmp/A/efaada395c22467da7016e016c858ca5, store: [table=TestAcidGuarantees family=A region=a2c58e0d0c97538318beef21ebb34095] 2024-11-27T12:27:01,043 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/.tmp/A/efaada395c22467da7016e016c858ca5 is 175, key is test_row_0/A:col10/1732710419483/Put/seqid=0 2024-11-27T12:27:01,046 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742377_1553 (size=31105) 2024-11-27T12:27:01,083 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-27T12:27:01,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095. 2024-11-27T12:27:01,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095. as already flushing 2024-11-27T12:27:01,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095. 2024-11-27T12:27:01,083 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => a2c58e0d0c97538318beef21ebb34095, NAME => 'TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:27:01,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => a2c58e0d0c97538318beef21ebb34095, NAME => 'TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:27:01,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36215 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a2c58e0d0c97538318beef21ebb34095, NAME => 'TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a2c58e0d0c97538318beef21ebb34095, NAME => 'TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:27:01,186 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/.tmp/A/0663abe7b9644947ba30247d80ae337d as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/A/0663abe7b9644947ba30247d80ae337d 2024-11-27T12:27:01,189 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in a2c58e0d0c97538318beef21ebb34095/A of a2c58e0d0c97538318beef21ebb34095 into 0663abe7b9644947ba30247d80ae337d(size=30.5 K), total size for store is 30.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T12:27:01,189 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a2c58e0d0c97538318beef21ebb34095: 2024-11-27T12:27:01,189 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., storeName=a2c58e0d0c97538318beef21ebb34095/A, priority=12, startTime=1732710420343; duration=0sec 2024-11-27T12:27:01,189 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T12:27:01,189 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a2c58e0d0c97538318beef21ebb34095:A 2024-11-27T12:27:01,235 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-27T12:27:01,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095. 2024-11-27T12:27:01,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095. as already flushing 2024-11-27T12:27:01,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095. 2024-11-27T12:27:01,236 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => a2c58e0d0c97538318beef21ebb34095, NAME => 'TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:27:01,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => a2c58e0d0c97538318beef21ebb34095, NAME => 'TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:27:01,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36215 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a2c58e0d0c97538318beef21ebb34095, NAME => 'TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a2c58e0d0c97538318beef21ebb34095, NAME => 'TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:27:01,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-27T12:27:01,309 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:27:01,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35496 deadline: 1732710481308, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:27:01,310 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:27:01,310 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:27:01,310 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:27:01,319 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:27:01,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35442 deadline: 1732710481318, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:27:01,320 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:27:01,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35492 deadline: 1732710481318, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:27:01,320 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:27:01,320 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:27:01,320 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:27:01,320 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:27:01,320 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:27:01,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35506 deadline: 1732710481318, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:27:01,320 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:27:01,320 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:27:01,320 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:27:01,320 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:27:01,320 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:27:01,388 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-27T12:27:01,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095. 2024-11-27T12:27:01,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095. as already flushing 2024-11-27T12:27:01,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095. 2024-11-27T12:27:01,389 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => a2c58e0d0c97538318beef21ebb34095, NAME => 'TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:27:01,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => a2c58e0d0c97538318beef21ebb34095, NAME => 'TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:27:01,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36215 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a2c58e0d0c97538318beef21ebb34095, NAME => 'TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a2c58e0d0c97538318beef21ebb34095, NAME => 'TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:27:01,447 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=137, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/.tmp/A/efaada395c22467da7016e016c858ca5 2024-11-27T12:27:01,452 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/.tmp/B/1eff988f7949427488836acf331b6a14 is 50, key is test_row_0/B:col10/1732710419483/Put/seqid=0 2024-11-27T12:27:01,455 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742378_1554 (size=12151) 2024-11-27T12:27:01,541 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-27T12:27:01,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095. 2024-11-27T12:27:01,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095. as already flushing 2024-11-27T12:27:01,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095. 2024-11-27T12:27:01,541 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => a2c58e0d0c97538318beef21ebb34095, NAME => 'TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:27:01,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => a2c58e0d0c97538318beef21ebb34095, NAME => 'TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:27:01,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36215 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a2c58e0d0c97538318beef21ebb34095, NAME => 'TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a2c58e0d0c97538318beef21ebb34095, NAME => 'TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:27:01,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-27T12:27:01,693 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-27T12:27:01,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095. 2024-11-27T12:27:01,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095. as already flushing 2024-11-27T12:27:01,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095. 2024-11-27T12:27:01,694 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => a2c58e0d0c97538318beef21ebb34095, NAME => 'TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:27:01,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => a2c58e0d0c97538318beef21ebb34095, NAME => 'TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:27:01,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36215 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a2c58e0d0c97538318beef21ebb34095, NAME => 'TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a2c58e0d0c97538318beef21ebb34095, NAME => 'TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:27:01,819 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:27:01,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35496 deadline: 1732710481818, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:27:01,819 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:27:01,819 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:27:01,819 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:27:01,829 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:27:01,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35442 deadline: 1732710481828, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:27:01,829 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:27:01,829 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:27:01,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35492 deadline: 1732710481828, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:27:01,830 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:27:01,830 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:27:01,830 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:27:01,830 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:27:01,830 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:27:01,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35506 deadline: 1732710481828, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:27:01,830 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:27:01,830 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:27:01,830 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:27:01,830 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:27:01,846 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-27T12:27:01,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095. 2024-11-27T12:27:01,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095. as already flushing 2024-11-27T12:27:01,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095. 2024-11-27T12:27:01,846 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => a2c58e0d0c97538318beef21ebb34095, NAME => 'TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:27:01,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => a2c58e0d0c97538318beef21ebb34095, NAME => 'TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:27:01,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36215 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a2c58e0d0c97538318beef21ebb34095, NAME => 'TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a2c58e0d0c97538318beef21ebb34095, NAME => 'TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:27:01,855 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=137 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/.tmp/B/1eff988f7949427488836acf331b6a14 2024-11-27T12:27:01,861 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/.tmp/C/91c2ab16cb8e4cd79010a5626c7fc4e3 is 50, key is test_row_0/C:col10/1732710419483/Put/seqid=0 2024-11-27T12:27:01,863 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742379_1555 (size=12151) 2024-11-27T12:27:01,998 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-27T12:27:01,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095. 2024-11-27T12:27:01,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095. as already flushing 2024-11-27T12:27:01,999 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095. 2024-11-27T12:27:01,999 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => a2c58e0d0c97538318beef21ebb34095, NAME => 'TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:27:01,999 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => a2c58e0d0c97538318beef21ebb34095, NAME => 'TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:27:01,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36215 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a2c58e0d0c97538318beef21ebb34095, NAME => 'TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a2c58e0d0c97538318beef21ebb34095, NAME => 'TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:27:02,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-27T12:27:02,151 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-27T12:27:02,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095. 2024-11-27T12:27:02,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095. as already flushing 2024-11-27T12:27:02,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095. 2024-11-27T12:27:02,151 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => a2c58e0d0c97538318beef21ebb34095, NAME => 'TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:27:02,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => a2c58e0d0c97538318beef21ebb34095, NAME => 'TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:27:02,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36215 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a2c58e0d0c97538318beef21ebb34095, NAME => 'TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a2c58e0d0c97538318beef21ebb34095, NAME => 'TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:27:02,264 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=137 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/.tmp/C/91c2ab16cb8e4cd79010a5626c7fc4e3 2024-11-27T12:27:02,267 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/.tmp/A/efaada395c22467da7016e016c858ca5 as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/A/efaada395c22467da7016e016c858ca5 2024-11-27T12:27:02,270 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/A/efaada395c22467da7016e016c858ca5, entries=150, sequenceid=137, filesize=30.4 K 2024-11-27T12:27:02,270 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/.tmp/B/1eff988f7949427488836acf331b6a14 as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/B/1eff988f7949427488836acf331b6a14 2024-11-27T12:27:02,273 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/B/1eff988f7949427488836acf331b6a14, entries=150, sequenceid=137, filesize=11.9 K 2024-11-27T12:27:02,273 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/.tmp/C/91c2ab16cb8e4cd79010a5626c7fc4e3 as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/C/91c2ab16cb8e4cd79010a5626c7fc4e3 2024-11-27T12:27:02,276 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/C/91c2ab16cb8e4cd79010a5626c7fc4e3, entries=150, sequenceid=137, filesize=11.9 K 2024-11-27T12:27:02,277 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=114.05 KB/116790 for a2c58e0d0c97538318beef21ebb34095 in 1647ms, sequenceid=137, compaction requested=false 2024-11-27T12:27:02,277 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a2c58e0d0c97538318beef21ebb34095: 2024-11-27T12:27:02,303 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-27T12:27:02,304 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095. 2024-11-27T12:27:02,304 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2902): Flushing a2c58e0d0c97538318beef21ebb34095 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-27T12:27:02,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127e5a1f0fec8824a73b09559be94e0d16e_a2c58e0d0c97538318beef21ebb34095 is 50, key is test_row_0/A:col10/1732710420679/Put/seqid=0 2024-11-27T12:27:02,313 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742380_1556 (size=12304) 2024-11-27T12:27:02,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:27:02,716 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127e5a1f0fec8824a73b09559be94e0d16e_a2c58e0d0c97538318beef21ebb34095 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127e5a1f0fec8824a73b09559be94e0d16e_a2c58e0d0c97538318beef21ebb34095 2024-11-27T12:27:02,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/.tmp/A/40eb2171a726414d976f9b21bf892db9, store: [table=TestAcidGuarantees family=A region=a2c58e0d0c97538318beef21ebb34095] 2024-11-27T12:27:02,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/.tmp/A/40eb2171a726414d976f9b21bf892db9 is 175, key is test_row_0/A:col10/1732710420679/Put/seqid=0 2024-11-27T12:27:02,720 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742381_1557 (size=31105) 2024-11-27T12:27:02,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(8855): Flush requested on a2c58e0d0c97538318beef21ebb34095 2024-11-27T12:27:02,830 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095. as already flushing 2024-11-27T12:27:02,853 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:27:02,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35496 deadline: 1732710482849, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:27:02,853 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:27:02,853 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:27:02,853 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:27:02,853 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:27:02,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35442 deadline: 1732710482849, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:27:02,853 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:27:02,853 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:27:02,853 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:27:02,853 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:27:02,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35492 deadline: 1732710482850, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:27:02,854 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:27:02,854 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:27:02,854 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:27:02,854 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:27:02,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35506 deadline: 1732710482851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:27:02,854 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:27:02,854 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:27:02,854 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:27:02,959 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:27:02,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35496 deadline: 1732710482957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:27:02,960 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:27:02,960 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:27:02,960 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:27:02,960 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:27:02,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35442 deadline: 1732710482958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:27:02,960 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:27:02,960 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:27:02,960 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:27:02,960 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:27:02,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35492 deadline: 1732710482958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:27:02,960 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:27:02,960 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:27:02,960 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:27:02,960 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:27:02,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35506 deadline: 1732710482958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:27:02,961 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:27:02,961 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:27:02,961 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:27:03,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-27T12:27:03,120 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=159, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/.tmp/A/40eb2171a726414d976f9b21bf892db9 2024-11-27T12:27:03,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/.tmp/B/7a795df9d9564cc3a94b814db6582ec8 is 50, key is test_row_0/B:col10/1732710420679/Put/seqid=0 2024-11-27T12:27:03,128 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742382_1558 (size=12151) 2024-11-27T12:27:03,170 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:27:03,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35496 deadline: 1732710483167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:27:03,170 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:27:03,170 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:27:03,170 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:27:03,170 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:27:03,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35442 deadline: 1732710483168, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:27:03,170 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:27:03,170 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:27:03,170 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:27:03,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35492 deadline: 1732710483168, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:27:03,170 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:27:03,171 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:27:03,171 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:27:03,171 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:27:03,171 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:27:03,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35506 deadline: 1732710483168, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:27:03,171 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:27:03,171 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:27:03,171 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:27:03,480 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:27:03,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35496 deadline: 1732710483478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:27:03,481 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:27:03,481 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:27:03,481 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:27:03,481 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:27:03,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35442 deadline: 1732710483478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:27:03,481 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:27:03,481 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:27:03,481 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:27:03,481 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:27:03,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35492 deadline: 1732710483478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:27:03,481 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:27:03,481 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:27:03,481 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:27:03,481 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:27:03,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35506 deadline: 1732710483478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:27:03,482 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:27:03,482 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:27:03,482 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:27:03,529 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=159 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/.tmp/B/7a795df9d9564cc3a94b814db6582ec8 2024-11-27T12:27:03,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/.tmp/C/e7c83f80aa514409a605b299d757738f is 50, key is test_row_0/C:col10/1732710420679/Put/seqid=0 2024-11-27T12:27:03,536 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742383_1559 (size=12151) 2024-11-27T12:27:03,937 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=159 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/.tmp/C/e7c83f80aa514409a605b299d757738f 2024-11-27T12:27:03,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/.tmp/A/40eb2171a726414d976f9b21bf892db9 as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/A/40eb2171a726414d976f9b21bf892db9 2024-11-27T12:27:03,943 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/A/40eb2171a726414d976f9b21bf892db9, entries=150, sequenceid=159, filesize=30.4 K 2024-11-27T12:27:03,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/.tmp/B/7a795df9d9564cc3a94b814db6582ec8 as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/B/7a795df9d9564cc3a94b814db6582ec8 2024-11-27T12:27:03,946 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/B/7a795df9d9564cc3a94b814db6582ec8, entries=150, sequenceid=159, filesize=11.9 K 2024-11-27T12:27:03,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/.tmp/C/e7c83f80aa514409a605b299d757738f as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/C/e7c83f80aa514409a605b299d757738f 2024-11-27T12:27:03,950 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/C/e7c83f80aa514409a605b299d757738f, entries=150, sequenceid=159, filesize=11.9 K 2024-11-27T12:27:03,950 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=87.22 KB/89310 for a2c58e0d0c97538318beef21ebb34095 in 1646ms, sequenceid=159, compaction requested=true 2024-11-27T12:27:03,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2603): Flush status journal for a2c58e0d0c97538318beef21ebb34095: 2024-11-27T12:27:03,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095. 2024-11-27T12:27:03,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=161 2024-11-27T12:27:03,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36215 {}] master.HMaster(4169): Remote procedure done, pid=161 2024-11-27T12:27:03,953 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=161, resume processing ppid=160 2024-11-27T12:27:03,953 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=161, ppid=160, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.0200 sec 2024-11-27T12:27:03,954 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=160, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees in 3.0240 sec 2024-11-27T12:27:03,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(8855): Flush requested on a2c58e0d0c97538318beef21ebb34095 2024-11-27T12:27:03,991 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T12:27:03,991 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a2c58e0d0c97538318beef21ebb34095 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-11-27T12:27:03,997 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127ac250f6c02cc4f50bdb53177d111f030_a2c58e0d0c97538318beef21ebb34095 is 50, key is test_row_0/A:col10/1732710423991/Put/seqid=0 2024-11-27T12:27:03,999 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742384_1560 (size=12304) 2024-11-27T12:27:04,009 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:27:04,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35496 deadline: 1732710484005, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:27:04,009 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:27:04,010 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:27:04,010 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:27:04,012 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:27:04,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35506 deadline: 1732710484007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:27:04,012 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:27:04,012 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:27:04,012 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:27:04,012 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:27:04,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35492 deadline: 1732710484008, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:27:04,013 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:27:04,013 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:27:04,013 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:27:04,013 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:27:04,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35442 deadline: 1732710484009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:27:04,013 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:27:04,013 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:27:04,013 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:27:04,120 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:27:04,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35496 deadline: 1732710484118, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:27:04,121 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:27:04,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35506 deadline: 1732710484118, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:27:04,121 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:27:04,121 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:27:04,121 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:27:04,121 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:27:04,121 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:27:04,121 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:27:04,121 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:27:04,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35492 deadline: 1732710484119, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:27:04,122 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:27:04,122 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:27:04,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35442 deadline: 1732710484119, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:27:04,122 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:27:04,122 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:27:04,122 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:27:04,122 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:27:04,122 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:27:04,330 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:27:04,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35496 deadline: 1732710484327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:27:04,331 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:27:04,331 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:27:04,331 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:27:04,331 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:27:04,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35506 deadline: 1732710484328, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:27:04,331 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:27:04,331 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:27:04,331 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:27:04,331 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:27:04,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35492 deadline: 1732710484328, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:27:04,331 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:27:04,331 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:27:04,331 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:27:04,331 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:27:04,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35442 deadline: 1732710484328, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:27:04,332 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:27:04,332 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:27:04,332 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:27:04,400 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:27:04,403 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127ac250f6c02cc4f50bdb53177d111f030_a2c58e0d0c97538318beef21ebb34095 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127ac250f6c02cc4f50bdb53177d111f030_a2c58e0d0c97538318beef21ebb34095 2024-11-27T12:27:04,404 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/.tmp/A/687530490fb947ffbc47a64e19e8b627, store: [table=TestAcidGuarantees family=A region=a2c58e0d0c97538318beef21ebb34095] 2024-11-27T12:27:04,404 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/.tmp/A/687530490fb947ffbc47a64e19e8b627 is 175, key is test_row_0/A:col10/1732710423991/Put/seqid=0 2024-11-27T12:27:04,407 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742385_1561 (size=31105) 2024-11-27T12:27:04,407 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=177, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/.tmp/A/687530490fb947ffbc47a64e19e8b627 2024-11-27T12:27:04,413 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/.tmp/B/116d99be4c994e6b9824db994bbcdef8 is 50, key is test_row_0/B:col10/1732710423991/Put/seqid=0 2024-11-27T12:27:04,415 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742386_1562 (size=12151) 2024-11-27T12:27:04,641 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:27:04,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35496 deadline: 1732710484638, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:27:04,641 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:27:04,641 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:27:04,641 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:27:04,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35506 deadline: 1732710484638, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:27:04,641 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:27:04,641 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:27:04,641 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:27:04,641 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:27:04,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35492 deadline: 1732710484638, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:27:04,641 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:27:04,641 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:27:04,641 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:27:04,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35442 deadline: 1732710484638, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:27:04,641 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:27:04,641 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:27:04,642 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:27:04,642 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:27:04,642 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:27:04,651 DEBUG [regionserver/1ba5af43fd99:0.Chore.1 {}] throttle.PressureAwareCompactionThroughputController(103): CompactionPressure is 0.0, tune throughput to 50.00 MB/second 2024-11-27T12:27:04,816 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=177 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/.tmp/B/116d99be4c994e6b9824db994bbcdef8 2024-11-27T12:27:04,822 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/.tmp/C/22ffae6bc9834fd1b1abf41f55c812d2 is 50, key is test_row_0/C:col10/1732710423991/Put/seqid=0 2024-11-27T12:27:04,824 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742387_1563 (size=12151) 2024-11-27T12:27:05,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-27T12:27:05,088 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T12:27:05,088 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36215 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-27T12:27:05,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36215 {}] procedure2.ProcedureExecutor(1139): Stored pid=162, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees 2024-11-27T12:27:05,090 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=162, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T12:27:05,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-27T12:27:05,090 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=162, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T12:27:05,090 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=163, ppid=162, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T12:27:05,151 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:27:05,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35496 deadline: 1732710485148, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:27:05,151 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:27:05,151 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:27:05,152 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:27:05,152 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:27:05,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35506 deadline: 1732710485148, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:27:05,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35492 deadline: 1732710485148, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:27:05,152 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:27:05,152 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:27:05,152 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:27:05,152 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:27:05,152 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:27:05,152 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:27:05,152 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:27:05,152 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:27:05,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35442 deadline: 1732710485148, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:27:05,152 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:27:05,152 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:27:05,152 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:27:05,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-27T12:27:05,225 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=177 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/.tmp/C/22ffae6bc9834fd1b1abf41f55c812d2 2024-11-27T12:27:05,228 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/.tmp/A/687530490fb947ffbc47a64e19e8b627 as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/A/687530490fb947ffbc47a64e19e8b627 2024-11-27T12:27:05,231 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/A/687530490fb947ffbc47a64e19e8b627, entries=150, sequenceid=177, filesize=30.4 K 2024-11-27T12:27:05,231 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/.tmp/B/116d99be4c994e6b9824db994bbcdef8 as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/B/116d99be4c994e6b9824db994bbcdef8 2024-11-27T12:27:05,234 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/B/116d99be4c994e6b9824db994bbcdef8, entries=150, sequenceid=177, filesize=11.9 K 2024-11-27T12:27:05,234 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/.tmp/C/22ffae6bc9834fd1b1abf41f55c812d2 as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/C/22ffae6bc9834fd1b1abf41f55c812d2 2024-11-27T12:27:05,237 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/C/22ffae6bc9834fd1b1abf41f55c812d2, entries=150, sequenceid=177, filesize=11.9 K 2024-11-27T12:27:05,237 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=100.63 KB/103050 for a2c58e0d0c97538318beef21ebb34095 in 1246ms, sequenceid=177, compaction requested=true 2024-11-27T12:27:05,237 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a2c58e0d0c97538318beef21ebb34095: 2024-11-27T12:27:05,237 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a2c58e0d0c97538318beef21ebb34095:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T12:27:05,237 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T12:27:05,237 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-27T12:27:05,237 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a2c58e0d0c97538318beef21ebb34095:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T12:27:05,237 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T12:27:05,237 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-27T12:27:05,238 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a2c58e0d0c97538318beef21ebb34095:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T12:27:05,238 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T12:27:05,238 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 124510 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-27T12:27:05,238 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48694 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-27T12:27:05,238 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HStore(1541): a2c58e0d0c97538318beef21ebb34095/B is initiating minor compaction (all files) 2024-11-27T12:27:05,238 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HStore(1541): a2c58e0d0c97538318beef21ebb34095/A is initiating minor compaction (all files) 2024-11-27T12:27:05,238 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a2c58e0d0c97538318beef21ebb34095/A in TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095. 2024-11-27T12:27:05,238 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a2c58e0d0c97538318beef21ebb34095/B in TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095. 2024-11-27T12:27:05,238 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/A/0663abe7b9644947ba30247d80ae337d, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/A/efaada395c22467da7016e016c858ca5, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/A/40eb2171a726414d976f9b21bf892db9, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/A/687530490fb947ffbc47a64e19e8b627] into tmpdir=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/.tmp, totalSize=121.6 K 2024-11-27T12:27:05,238 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/B/416010e191074cfd839210937a8bae9e, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/B/1eff988f7949427488836acf331b6a14, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/B/7a795df9d9564cc3a94b814db6582ec8, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/B/116d99be4c994e6b9824db994bbcdef8] into tmpdir=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/.tmp, totalSize=47.6 K 2024-11-27T12:27:05,238 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095. 2024-11-27T12:27:05,238 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095. files: [hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/A/0663abe7b9644947ba30247d80ae337d, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/A/efaada395c22467da7016e016c858ca5, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/A/40eb2171a726414d976f9b21bf892db9, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/A/687530490fb947ffbc47a64e19e8b627] 2024-11-27T12:27:05,239 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.Compactor(225): Compacting 416010e191074cfd839210937a8bae9e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1732710419419 2024-11-27T12:27:05,239 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0663abe7b9644947ba30247d80ae337d, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1732710419419 2024-11-27T12:27:05,239 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] compactions.Compactor(225): Compacting efaada395c22467da7016e016c858ca5, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=137, earliestPutTs=1732710419483 2024-11-27T12:27:05,239 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.Compactor(225): Compacting 1eff988f7949427488836acf331b6a14, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=137, earliestPutTs=1732710419483 2024-11-27T12:27:05,239 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.Compactor(225): Compacting 7a795df9d9564cc3a94b814db6582ec8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=159, earliestPutTs=1732710420678 2024-11-27T12:27:05,239 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] compactions.Compactor(225): Compacting 40eb2171a726414d976f9b21bf892db9, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=159, earliestPutTs=1732710420678 2024-11-27T12:27:05,239 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.Compactor(225): Compacting 116d99be4c994e6b9824db994bbcdef8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=177, earliestPutTs=1732710422849 2024-11-27T12:27:05,239 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] compactions.Compactor(225): Compacting 687530490fb947ffbc47a64e19e8b627, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=177, earliestPutTs=1732710422849 2024-11-27T12:27:05,242 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-27T12:27:05,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095. 2024-11-27T12:27:05,242 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2902): Flushing a2c58e0d0c97538318beef21ebb34095 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-11-27T12:27:05,244 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=a2c58e0d0c97538318beef21ebb34095] 2024-11-27T12:27:05,246 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a2c58e0d0c97538318beef21ebb34095#B#compaction#470 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-27T12:27:05,246 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/.tmp/B/4401eefbca9842348470a50f6d4fbc51 is 50, key is test_row_0/B:col10/1732710423991/Put/seqid=0 2024-11-27T12:27:05,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127823ad32546f04bbc9b8bf3e9eafd1a19_a2c58e0d0c97538318beef21ebb34095 is 50, key is test_row_0/A:col10/1732710423999/Put/seqid=0 2024-11-27T12:27:05,247 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411271ab5e3c986a248a38e335ef8a5850355_a2c58e0d0c97538318beef21ebb34095 store=[table=TestAcidGuarantees family=A region=a2c58e0d0c97538318beef21ebb34095] 2024-11-27T12:27:05,249 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411271ab5e3c986a248a38e335ef8a5850355_a2c58e0d0c97538318beef21ebb34095, store=[table=TestAcidGuarantees family=A region=a2c58e0d0c97538318beef21ebb34095] 2024-11-27T12:27:05,250 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411271ab5e3c986a248a38e335ef8a5850355_a2c58e0d0c97538318beef21ebb34095 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=a2c58e0d0c97538318beef21ebb34095] 2024-11-27T12:27:05,251 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742389_1565 (size=12304) 2024-11-27T12:27:05,254 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742388_1564 (size=12527) 2024-11-27T12:27:05,254 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742390_1566 (size=4469) 2024-11-27T12:27:05,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-27T12:27:05,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:27:05,655 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a2c58e0d0c97538318beef21ebb34095#A#compaction#471 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T12:27:05,655 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/.tmp/A/9c726caa786e4965a39b59b3effb0c33 is 175, key is test_row_0/A:col10/1732710423991/Put/seqid=0 2024-11-27T12:27:05,655 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127823ad32546f04bbc9b8bf3e9eafd1a19_a2c58e0d0c97538318beef21ebb34095 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127823ad32546f04bbc9b8bf3e9eafd1a19_a2c58e0d0c97538318beef21ebb34095 2024-11-27T12:27:05,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/.tmp/A/5cb5972502034a0baea3f254a5ef1fa0, store: [table=TestAcidGuarantees family=A region=a2c58e0d0c97538318beef21ebb34095] 2024-11-27T12:27:05,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/.tmp/A/5cb5972502034a0baea3f254a5ef1fa0 is 175, key is test_row_0/A:col10/1732710423999/Put/seqid=0 2024-11-27T12:27:05,658 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/.tmp/B/4401eefbca9842348470a50f6d4fbc51 as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/B/4401eefbca9842348470a50f6d4fbc51 2024-11-27T12:27:05,659 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742391_1567 (size=31481) 2024-11-27T12:27:05,664 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742392_1568 (size=31105) 2024-11-27T12:27:05,664 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=195, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/.tmp/A/5cb5972502034a0baea3f254a5ef1fa0 2024-11-27T12:27:05,666 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in a2c58e0d0c97538318beef21ebb34095/B of a2c58e0d0c97538318beef21ebb34095 into 4401eefbca9842348470a50f6d4fbc51(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T12:27:05,666 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a2c58e0d0c97538318beef21ebb34095: 2024-11-27T12:27:05,666 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., storeName=a2c58e0d0c97538318beef21ebb34095/B, priority=12, startTime=1732710425237; duration=0sec 2024-11-27T12:27:05,666 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T12:27:05,666 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a2c58e0d0c97538318beef21ebb34095:B 2024-11-27T12:27:05,666 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-27T12:27:05,671 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48694 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-27T12:27:05,671 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HStore(1541): a2c58e0d0c97538318beef21ebb34095/C is initiating minor compaction (all files) 2024-11-27T12:27:05,672 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a2c58e0d0c97538318beef21ebb34095/C in TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095. 2024-11-27T12:27:05,672 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/C/7fa7e87809084578bb0b283ab73c8d84, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/C/91c2ab16cb8e4cd79010a5626c7fc4e3, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/C/e7c83f80aa514409a605b299d757738f, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/C/22ffae6bc9834fd1b1abf41f55c812d2] into tmpdir=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/.tmp, totalSize=47.6 K 2024-11-27T12:27:05,672 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.Compactor(225): Compacting 7fa7e87809084578bb0b283ab73c8d84, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1732710419419 2024-11-27T12:27:05,672 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.Compactor(225): Compacting 91c2ab16cb8e4cd79010a5626c7fc4e3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=137, earliestPutTs=1732710419483 2024-11-27T12:27:05,672 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.Compactor(225): Compacting e7c83f80aa514409a605b299d757738f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=159, earliestPutTs=1732710420678 2024-11-27T12:27:05,673 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.Compactor(225): Compacting 22ffae6bc9834fd1b1abf41f55c812d2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=177, earliestPutTs=1732710422849 2024-11-27T12:27:05,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/.tmp/B/2d9905f5b3c9464394eec2ed540ed6ac is 50, key is test_row_0/B:col10/1732710423999/Put/seqid=0 2024-11-27T12:27:05,687 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a2c58e0d0c97538318beef21ebb34095#C#compaction#474 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T12:27:05,687 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/.tmp/C/66e7f70f029b4c9c9a7137cd07a83733 is 50, key is test_row_0/C:col10/1732710423991/Put/seqid=0 2024-11-27T12:27:05,689 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742393_1569 (size=12151) 2024-11-27T12:27:05,691 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742394_1570 (size=12527) 2024-11-27T12:27:05,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-27T12:27:06,063 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/.tmp/A/9c726caa786e4965a39b59b3effb0c33 as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/A/9c726caa786e4965a39b59b3effb0c33 2024-11-27T12:27:06,066 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in a2c58e0d0c97538318beef21ebb34095/A of a2c58e0d0c97538318beef21ebb34095 into 9c726caa786e4965a39b59b3effb0c33(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T12:27:06,066 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a2c58e0d0c97538318beef21ebb34095: 2024-11-27T12:27:06,066 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., storeName=a2c58e0d0c97538318beef21ebb34095/A, priority=12, startTime=1732710425237; duration=0sec 2024-11-27T12:27:06,066 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T12:27:06,066 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a2c58e0d0c97538318beef21ebb34095:A 2024-11-27T12:27:06,090 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=195 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/.tmp/B/2d9905f5b3c9464394eec2ed540ed6ac 2024-11-27T12:27:06,094 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/.tmp/C/66e7f70f029b4c9c9a7137cd07a83733 as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/C/66e7f70f029b4c9c9a7137cd07a83733 2024-11-27T12:27:06,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/.tmp/C/8d3207247622435aac8158df1d185eae is 50, key is test_row_0/C:col10/1732710423999/Put/seqid=0 2024-11-27T12:27:06,100 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in a2c58e0d0c97538318beef21ebb34095/C of a2c58e0d0c97538318beef21ebb34095 into 66e7f70f029b4c9c9a7137cd07a83733(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T12:27:06,100 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a2c58e0d0c97538318beef21ebb34095: 2024-11-27T12:27:06,100 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., storeName=a2c58e0d0c97538318beef21ebb34095/C, priority=12, startTime=1732710425237; duration=0sec 2024-11-27T12:27:06,100 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T12:27:06,100 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a2c58e0d0c97538318beef21ebb34095:C 2024-11-27T12:27:06,101 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742395_1571 (size=12151) 2024-11-27T12:27:06,160 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095. as already flushing 2024-11-27T12:27:06,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(8855): Flush requested on a2c58e0d0c97538318beef21ebb34095 2024-11-27T12:27:06,181 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:27:06,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35506 deadline: 1732710486177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:27:06,182 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:27:06,182 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:27:06,182 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:27:06,182 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:27:06,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35442 deadline: 1732710486177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:27:06,182 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:27:06,182 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:27:06,182 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:27:06,184 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:27:06,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35496 deadline: 1732710486181, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:27:06,184 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:27:06,184 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:27:06,184 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:27:06,185 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:27:06,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35492 deadline: 1732710486181, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:27:06,185 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:27:06,185 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:27:06,185 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:27:06,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-27T12:27:06,290 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:27:06,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35506 deadline: 1732710486288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:27:06,290 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:27:06,290 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:27:06,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35442 deadline: 1732710486288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:27:06,290 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:27:06,290 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:27:06,290 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:27:06,290 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:27:06,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35496 deadline: 1732710486288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:27:06,290 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:27:06,291 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:27:06,291 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:27:06,291 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:27:06,291 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:27:06,291 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:27:06,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35492 deadline: 1732710486288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:27:06,291 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:27:06,291 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:27:06,291 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:27:06,501 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:27:06,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35506 deadline: 1732710486498, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:27:06,501 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=195 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/.tmp/C/8d3207247622435aac8158df1d185eae 2024-11-27T12:27:06,502 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:27:06,502 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:27:06,502 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:27:06,502 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:27:06,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35442 deadline: 1732710486498, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:27:06,502 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:27:06,502 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:27:06,502 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:27:06,502 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:27:06,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35496 deadline: 1732710486498, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:27:06,502 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:27:06,502 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:27:06,502 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:27:06,502 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:27:06,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35492 deadline: 1732710486498, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:27:06,503 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:27:06,503 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:27:06,503 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:27:06,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/.tmp/A/5cb5972502034a0baea3f254a5ef1fa0 as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/A/5cb5972502034a0baea3f254a5ef1fa0 2024-11-27T12:27:06,508 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/A/5cb5972502034a0baea3f254a5ef1fa0, entries=150, sequenceid=195, filesize=30.4 K 2024-11-27T12:27:06,508 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/.tmp/B/2d9905f5b3c9464394eec2ed540ed6ac as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/B/2d9905f5b3c9464394eec2ed540ed6ac 2024-11-27T12:27:06,511 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/B/2d9905f5b3c9464394eec2ed540ed6ac, entries=150, sequenceid=195, filesize=11.9 K 2024-11-27T12:27:06,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/.tmp/C/8d3207247622435aac8158df1d185eae as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/C/8d3207247622435aac8158df1d185eae 2024-11-27T12:27:06,514 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/C/8d3207247622435aac8158df1d185eae, entries=150, sequenceid=195, filesize=11.9 K 2024-11-27T12:27:06,515 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=107.34 KB/109920 for a2c58e0d0c97538318beef21ebb34095 in 1273ms, sequenceid=195, compaction requested=false 2024-11-27T12:27:06,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2603): Flush status journal for a2c58e0d0c97538318beef21ebb34095: 2024-11-27T12:27:06,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095. 2024-11-27T12:27:06,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=163 2024-11-27T12:27:06,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36215 {}] master.HMaster(4169): Remote procedure done, pid=163 2024-11-27T12:27:06,518 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=163, resume processing ppid=162 2024-11-27T12:27:06,518 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=163, ppid=162, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4260 sec 2024-11-27T12:27:06,519 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=162, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees in 1.4300 sec 2024-11-27T12:27:06,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(8855): Flush requested on a2c58e0d0c97538318beef21ebb34095 2024-11-27T12:27:06,811 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T12:27:06,811 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a2c58e0d0c97538318beef21ebb34095 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-27T12:27:06,817 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127e915390bfe974e05a2b056f275272e00_a2c58e0d0c97538318beef21ebb34095 is 50, key is test_row_0/A:col10/1732710426811/Put/seqid=0 2024-11-27T12:27:06,820 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742396_1572 (size=14794) 2024-11-27T12:27:06,826 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:27:06,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35496 deadline: 1732710486822, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:27:06,826 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:27:06,826 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:27:06,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35506 deadline: 1732710486823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:27:06,826 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:27:06,826 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:27:06,826 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:27:06,826 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:27:06,826 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:27:06,829 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:27:06,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35442 deadline: 1732710486826, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:27:06,829 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:27:06,829 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:27:06,829 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:27:06,829 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:27:06,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35492 deadline: 1732710486826, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:27:06,830 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:27:06,830 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:27:06,830 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:27:06,929 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:27:06,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35496 deadline: 1732710486927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:27:06,929 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:27:06,929 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:27:06,929 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:27:06,929 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:27:06,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35506 deadline: 1732710486927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:27:06,930 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:27:06,930 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:27:06,930 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:27:06,939 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:27:06,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35442 deadline: 1732710486938, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:27:06,939 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:27:06,939 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:27:06,939 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:27:06,940 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:27:06,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35492 deadline: 1732710486938, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:27:06,940 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:27:06,940 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:27:06,941 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:27:07,139 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:27:07,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35496 deadline: 1732710487138, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:27:07,140 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:27:07,140 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:27:07,140 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:27:07,140 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:27:07,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35506 deadline: 1732710487138, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:27:07,140 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:27:07,140 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:27:07,140 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:27:07,151 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:27:07,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35442 deadline: 1732710487148, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:27:07,151 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:27:07,151 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:27:07,151 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:27:07,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35492 deadline: 1732710487148, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:27:07,151 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:27:07,151 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:27:07,151 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:27:07,151 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:27:07,220 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:27:07,224 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127e915390bfe974e05a2b056f275272e00_a2c58e0d0c97538318beef21ebb34095 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127e915390bfe974e05a2b056f275272e00_a2c58e0d0c97538318beef21ebb34095 2024-11-27T12:27:07,224 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/.tmp/A/78d4650d42b54761a5d81b875b0f7add, store: [table=TestAcidGuarantees family=A region=a2c58e0d0c97538318beef21ebb34095] 2024-11-27T12:27:07,225 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/.tmp/A/78d4650d42b54761a5d81b875b0f7add is 175, key is test_row_0/A:col10/1732710426811/Put/seqid=0 2024-11-27T12:27:07,229 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742397_1573 (size=39749) 2024-11-27T12:27:07,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-27T12:27:07,238 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T12:27:07,238 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36215 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-27T12:27:07,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36215 {}] procedure2.ProcedureExecutor(1139): Stored pid=164, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees 2024-11-27T12:27:07,239 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=164, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T12:27:07,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-27T12:27:07,240 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=164, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T12:27:07,240 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=165, ppid=164, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T12:27:07,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-27T12:27:07,392 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-27T12:27:07,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095. 2024-11-27T12:27:07,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095. as already flushing 2024-11-27T12:27:07,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095. 2024-11-27T12:27:07,392 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => a2c58e0d0c97538318beef21ebb34095, NAME => 'TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:27:07,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => a2c58e0d0c97538318beef21ebb34095, NAME => 'TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:27:07,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36215 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a2c58e0d0c97538318beef21ebb34095, NAME => 'TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a2c58e0d0c97538318beef21ebb34095, NAME => 'TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:27:07,450 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:27:07,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35496 deadline: 1732710487448, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:27:07,450 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:27:07,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35506 deadline: 1732710487448, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:27:07,450 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:27:07,450 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:27:07,450 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:27:07,451 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:27:07,451 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:27:07,451 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:27:07,459 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:27:07,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35442 deadline: 1732710487457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:27:07,460 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:27:07,460 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:27:07,460 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:27:07,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35492 deadline: 1732710487458, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:27:07,460 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:27:07,460 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:27:07,460 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:27:07,460 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:27:07,545 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-27T12:27:07,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095. 2024-11-27T12:27:07,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095. as already flushing 2024-11-27T12:27:07,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095. 2024-11-27T12:27:07,545 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => a2c58e0d0c97538318beef21ebb34095, NAME => 'TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:27:07,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => a2c58e0d0c97538318beef21ebb34095, NAME => 'TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:27:07,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36215 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a2c58e0d0c97538318beef21ebb34095, NAME => 'TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a2c58e0d0c97538318beef21ebb34095, NAME => 'TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:27:07,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-27T12:27:07,630 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=220, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/.tmp/A/78d4650d42b54761a5d81b875b0f7add 2024-11-27T12:27:07,635 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/.tmp/B/2f83a6476b6942f5aa1def86cf7ec5f3 is 50, key is test_row_0/B:col10/1732710426811/Put/seqid=0 2024-11-27T12:27:07,637 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742398_1574 (size=12151) 2024-11-27T12:27:07,638 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=220 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/.tmp/B/2f83a6476b6942f5aa1def86cf7ec5f3 2024-11-27T12:27:07,643 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/.tmp/C/f1e4a25c3d4949c991002be4c832d312 is 50, key is test_row_0/C:col10/1732710426811/Put/seqid=0 2024-11-27T12:27:07,645 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742399_1575 (size=12151) 2024-11-27T12:27:07,696 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-27T12:27:07,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095. 2024-11-27T12:27:07,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095. as already flushing 2024-11-27T12:27:07,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095. 2024-11-27T12:27:07,696 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => a2c58e0d0c97538318beef21ebb34095, NAME => 'TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:27:07,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => a2c58e0d0c97538318beef21ebb34095, NAME => 'TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:27:07,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36215 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a2c58e0d0c97538318beef21ebb34095, NAME => 'TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a2c58e0d0c97538318beef21ebb34095, NAME => 'TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:27:07,849 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-27T12:27:07,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095. 2024-11-27T12:27:07,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095. as already flushing 2024-11-27T12:27:07,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095. 2024-11-27T12:27:07,849 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => a2c58e0d0c97538318beef21ebb34095, NAME => 'TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:27:07,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => a2c58e0d0c97538318beef21ebb34095, NAME => 'TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:27:07,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36215 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a2c58e0d0c97538318beef21ebb34095, NAME => 'TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a2c58e0d0c97538318beef21ebb34095, NAME => 'TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:27:07,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-27T12:27:07,960 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:27:07,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35496 deadline: 1732710487957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:27:07,960 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:27:07,960 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:27:07,960 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:27:07,960 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:27:07,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35506 deadline: 1732710487958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:27:07,960 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:27:07,960 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:27:07,960 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:27:07,971 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:27:07,971 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:27:07,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35442 deadline: 1732710487968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:27:07,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35492 deadline: 1732710487968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:27:07,972 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:27:07,972 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:27:07,972 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:27:07,972 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:27:07,972 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:27:07,972 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:27:08,001 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-27T12:27:08,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095. 2024-11-27T12:27:08,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095. as already flushing 2024-11-27T12:27:08,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095. 2024-11-27T12:27:08,002 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => a2c58e0d0c97538318beef21ebb34095, NAME => 'TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:27:08,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => a2c58e0d0c97538318beef21ebb34095, NAME => 'TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:27:08,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36215 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a2c58e0d0c97538318beef21ebb34095, NAME => 'TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a2c58e0d0c97538318beef21ebb34095, NAME => 'TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:27:08,046 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=220 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/.tmp/C/f1e4a25c3d4949c991002be4c832d312 2024-11-27T12:27:08,049 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/.tmp/A/78d4650d42b54761a5d81b875b0f7add as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/A/78d4650d42b54761a5d81b875b0f7add 2024-11-27T12:27:08,052 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/A/78d4650d42b54761a5d81b875b0f7add, entries=200, sequenceid=220, filesize=38.8 K 2024-11-27T12:27:08,052 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/.tmp/B/2f83a6476b6942f5aa1def86cf7ec5f3 as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/B/2f83a6476b6942f5aa1def86cf7ec5f3 2024-11-27T12:27:08,055 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/B/2f83a6476b6942f5aa1def86cf7ec5f3, entries=150, sequenceid=220, filesize=11.9 K 2024-11-27T12:27:08,055 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/.tmp/C/f1e4a25c3d4949c991002be4c832d312 as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/C/f1e4a25c3d4949c991002be4c832d312 2024-11-27T12:27:08,058 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/C/f1e4a25c3d4949c991002be4c832d312, entries=150, sequenceid=220, filesize=11.9 K 2024-11-27T12:27:08,059 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=80.51 KB/82440 for a2c58e0d0c97538318beef21ebb34095 in 1248ms, sequenceid=220, compaction requested=true 2024-11-27T12:27:08,059 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a2c58e0d0c97538318beef21ebb34095: 2024-11-27T12:27:08,059 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a2c58e0d0c97538318beef21ebb34095:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T12:27:08,059 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T12:27:08,059 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T12:27:08,059 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T12:27:08,060 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36829 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T12:27:08,060 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102335 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T12:27:08,060 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HStore(1541): a2c58e0d0c97538318beef21ebb34095/A is initiating minor compaction (all files) 2024-11-27T12:27:08,060 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HStore(1541): a2c58e0d0c97538318beef21ebb34095/B is initiating minor compaction (all files) 2024-11-27T12:27:08,060 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a2c58e0d0c97538318beef21ebb34095/A in TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095. 2024-11-27T12:27:08,060 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a2c58e0d0c97538318beef21ebb34095/B in TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095. 2024-11-27T12:27:08,060 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/B/4401eefbca9842348470a50f6d4fbc51, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/B/2d9905f5b3c9464394eec2ed540ed6ac, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/B/2f83a6476b6942f5aa1def86cf7ec5f3] into tmpdir=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/.tmp, totalSize=36.0 K 2024-11-27T12:27:08,060 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/A/9c726caa786e4965a39b59b3effb0c33, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/A/5cb5972502034a0baea3f254a5ef1fa0, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/A/78d4650d42b54761a5d81b875b0f7add] into tmpdir=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/.tmp, totalSize=99.9 K 2024-11-27T12:27:08,060 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095. 2024-11-27T12:27:08,060 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095. files: [hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/A/9c726caa786e4965a39b59b3effb0c33, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/A/5cb5972502034a0baea3f254a5ef1fa0, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/A/78d4650d42b54761a5d81b875b0f7add] 2024-11-27T12:27:08,061 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.Compactor(225): Compacting 4401eefbca9842348470a50f6d4fbc51, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=177, earliestPutTs=1732710422849 2024-11-27T12:27:08,061 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9c726caa786e4965a39b59b3effb0c33, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=177, earliestPutTs=1732710422849 2024-11-27T12:27:08,061 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5cb5972502034a0baea3f254a5ef1fa0, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1732710423999 2024-11-27T12:27:08,061 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.Compactor(225): Compacting 2d9905f5b3c9464394eec2ed540ed6ac, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1732710423999 2024-11-27T12:27:08,061 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] compactions.Compactor(225): Compacting 78d4650d42b54761a5d81b875b0f7add, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=220, earliestPutTs=1732710426180 2024-11-27T12:27:08,061 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.Compactor(225): Compacting 2f83a6476b6942f5aa1def86cf7ec5f3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=220, earliestPutTs=1732710426810 2024-11-27T12:27:08,064 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a2c58e0d0c97538318beef21ebb34095:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T12:27:08,064 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T12:27:08,064 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a2c58e0d0c97538318beef21ebb34095:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T12:27:08,064 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T12:27:08,065 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=a2c58e0d0c97538318beef21ebb34095] 2024-11-27T12:27:08,066 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a2c58e0d0c97538318beef21ebb34095#B#compaction#479 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-27T12:27:08,067 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/.tmp/B/00180eca120a42c584a9ef9d5dabed0f is 50, key is test_row_0/B:col10/1732710426811/Put/seqid=0 2024-11-27T12:27:08,068 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241127dcdbfb681eae451389e51b55ed2408de_a2c58e0d0c97538318beef21ebb34095 store=[table=TestAcidGuarantees family=A region=a2c58e0d0c97538318beef21ebb34095] 2024-11-27T12:27:08,070 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241127dcdbfb681eae451389e51b55ed2408de_a2c58e0d0c97538318beef21ebb34095, store=[table=TestAcidGuarantees family=A region=a2c58e0d0c97538318beef21ebb34095] 2024-11-27T12:27:08,070 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127dcdbfb681eae451389e51b55ed2408de_a2c58e0d0c97538318beef21ebb34095 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=a2c58e0d0c97538318beef21ebb34095] 2024-11-27T12:27:08,075 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742400_1576 (size=12629) 2024-11-27T12:27:08,076 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742401_1577 (size=4469) 2024-11-27T12:27:08,076 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a2c58e0d0c97538318beef21ebb34095#A#compaction#480 average throughput is 2.22 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T12:27:08,077 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/.tmp/A/32aa342608ca41c3ae55c681a24811af is 175, key is test_row_0/A:col10/1732710426811/Put/seqid=0 2024-11-27T12:27:08,079 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742402_1578 (size=31583) 2024-11-27T12:27:08,154 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-27T12:27:08,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095. 2024-11-27T12:27:08,154 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2902): Flushing a2c58e0d0c97538318beef21ebb34095 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-27T12:27:08,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112744ef427e875a41bfaea84833ae563daa_a2c58e0d0c97538318beef21ebb34095 is 50, key is test_row_0/A:col10/1732710426825/Put/seqid=0 2024-11-27T12:27:08,162 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742403_1579 (size=12304) 2024-11-27T12:27:08,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-27T12:27:08,479 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/.tmp/B/00180eca120a42c584a9ef9d5dabed0f as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/B/00180eca120a42c584a9ef9d5dabed0f 2024-11-27T12:27:08,483 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/.tmp/A/32aa342608ca41c3ae55c681a24811af as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/A/32aa342608ca41c3ae55c681a24811af 2024-11-27T12:27:08,484 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a2c58e0d0c97538318beef21ebb34095/B of a2c58e0d0c97538318beef21ebb34095 into 00180eca120a42c584a9ef9d5dabed0f(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T12:27:08,484 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a2c58e0d0c97538318beef21ebb34095: 2024-11-27T12:27:08,484 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., storeName=a2c58e0d0c97538318beef21ebb34095/B, priority=13, startTime=1732710428059; duration=0sec 2024-11-27T12:27:08,484 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T12:27:08,484 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a2c58e0d0c97538318beef21ebb34095:B 2024-11-27T12:27:08,484 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T12:27:08,485 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36829 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T12:27:08,485 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HStore(1541): a2c58e0d0c97538318beef21ebb34095/C is initiating minor compaction (all files) 2024-11-27T12:27:08,485 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a2c58e0d0c97538318beef21ebb34095/C in TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095. 2024-11-27T12:27:08,485 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/C/66e7f70f029b4c9c9a7137cd07a83733, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/C/8d3207247622435aac8158df1d185eae, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/C/f1e4a25c3d4949c991002be4c832d312] into tmpdir=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/.tmp, totalSize=36.0 K 2024-11-27T12:27:08,485 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.Compactor(225): Compacting 66e7f70f029b4c9c9a7137cd07a83733, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=177, earliestPutTs=1732710422849 2024-11-27T12:27:08,486 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.Compactor(225): Compacting 8d3207247622435aac8158df1d185eae, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1732710423999 2024-11-27T12:27:08,486 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.Compactor(225): Compacting f1e4a25c3d4949c991002be4c832d312, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=220, earliestPutTs=1732710426810 2024-11-27T12:27:08,487 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a2c58e0d0c97538318beef21ebb34095/A of a2c58e0d0c97538318beef21ebb34095 into 32aa342608ca41c3ae55c681a24811af(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T12:27:08,487 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a2c58e0d0c97538318beef21ebb34095: 2024-11-27T12:27:08,487 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., storeName=a2c58e0d0c97538318beef21ebb34095/A, priority=13, startTime=1732710428059; duration=0sec 2024-11-27T12:27:08,487 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T12:27:08,487 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a2c58e0d0c97538318beef21ebb34095:A 2024-11-27T12:27:08,491 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a2c58e0d0c97538318beef21ebb34095#C#compaction#482 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T12:27:08,492 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/.tmp/C/962bf29d5ff04658bab0af426812e61b is 50, key is test_row_0/C:col10/1732710426811/Put/seqid=0 2024-11-27T12:27:08,497 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742404_1580 (size=12629) 2024-11-27T12:27:08,563 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:27:08,566 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112744ef427e875a41bfaea84833ae563daa_a2c58e0d0c97538318beef21ebb34095 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112744ef427e875a41bfaea84833ae563daa_a2c58e0d0c97538318beef21ebb34095 2024-11-27T12:27:08,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/.tmp/A/0cba606e06304f04bcba15f05dedb95f, store: [table=TestAcidGuarantees family=A region=a2c58e0d0c97538318beef21ebb34095] 2024-11-27T12:27:08,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/.tmp/A/0cba606e06304f04bcba15f05dedb95f is 175, key is test_row_0/A:col10/1732710426825/Put/seqid=0 2024-11-27T12:27:08,581 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742405_1581 (size=31105) 2024-11-27T12:27:08,901 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/.tmp/C/962bf29d5ff04658bab0af426812e61b as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/C/962bf29d5ff04658bab0af426812e61b 2024-11-27T12:27:08,904 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a2c58e0d0c97538318beef21ebb34095/C of a2c58e0d0c97538318beef21ebb34095 into 962bf29d5ff04658bab0af426812e61b(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T12:27:08,904 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a2c58e0d0c97538318beef21ebb34095: 2024-11-27T12:27:08,904 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., storeName=a2c58e0d0c97538318beef21ebb34095/C, priority=13, startTime=1732710428064; duration=0sec 2024-11-27T12:27:08,904 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T12:27:08,904 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a2c58e0d0c97538318beef21ebb34095:C 2024-11-27T12:27:08,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(8855): Flush requested on a2c58e0d0c97538318beef21ebb34095 2024-11-27T12:27:08,970 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095. as already flushing 2024-11-27T12:27:08,982 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=235, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/.tmp/A/0cba606e06304f04bcba15f05dedb95f 2024-11-27T12:27:08,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/.tmp/B/88bdec78876e48c79190c8c2d6ceb92e is 50, key is test_row_0/B:col10/1732710426825/Put/seqid=0 2024-11-27T12:27:08,991 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742406_1582 (size=12151) 2024-11-27T12:27:08,995 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:27:08,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35492 deadline: 1732710488991, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:27:08,996 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:27:08,996 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:27:08,996 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:27:08,997 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:27:08,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35442 deadline: 1732710488993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:27:08,998 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:27:08,998 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:27:08,998 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:27:08,998 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:27:08,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35506 deadline: 1732710488994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:27:08,998 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:27:08,998 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:27:08,998 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:27:08,998 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:27:08,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35496 deadline: 1732710488995, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:27:08,999 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:27:08,999 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:27:08,999 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:27:09,100 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:27:09,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35492 deadline: 1732710489097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:27:09,100 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:27:09,100 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:27:09,100 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:27:09,109 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:27:09,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35442 deadline: 1732710489108, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:27:09,110 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:27:09,110 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:27:09,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35496 deadline: 1732710489108, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:27:09,110 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:27:09,110 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:27:09,110 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:27:09,110 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:27:09,110 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:27:09,110 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:27:09,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35506 deadline: 1732710489108, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:27:09,110 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:27:09,110 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:27:09,110 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:27:09,310 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:27:09,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35492 deadline: 1732710489308, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:27:09,311 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:27:09,311 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:27:09,311 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:27:09,320 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:27:09,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35442 deadline: 1732710489318, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:27:09,320 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:27:09,320 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:27:09,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35496 deadline: 1732710489318, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:27:09,320 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:27:09,320 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:27:09,320 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:27:09,320 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:27:09,320 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:27:09,320 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:27:09,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35506 deadline: 1732710489318, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:27:09,321 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:27:09,321 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:27:09,321 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:27:09,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-27T12:27:09,391 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=235 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/.tmp/B/88bdec78876e48c79190c8c2d6ceb92e 2024-11-27T12:27:09,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/.tmp/C/8bab4586020d4d5984d6bb534d889caa is 50, key is test_row_0/C:col10/1732710426825/Put/seqid=0 2024-11-27T12:27:09,399 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742407_1583 (size=12151) 2024-11-27T12:27:09,620 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:27:09,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35492 deadline: 1732710489618, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:27:09,620 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:27:09,620 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:27:09,620 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:27:09,630 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:27:09,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35442 deadline: 1732710489628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:27:09,630 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:27:09,630 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:27:09,630 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:27:09,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35496 deadline: 1732710489628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:27:09,630 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:27:09,631 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:27:09,631 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:27:09,631 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:27:09,631 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:27:09,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35506 deadline: 1732710489628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:27:09,631 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:27:09,631 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:27:09,631 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:27:09,799 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=235 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/.tmp/C/8bab4586020d4d5984d6bb534d889caa 2024-11-27T12:27:09,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/.tmp/A/0cba606e06304f04bcba15f05dedb95f as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/A/0cba606e06304f04bcba15f05dedb95f 2024-11-27T12:27:09,806 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/A/0cba606e06304f04bcba15f05dedb95f, entries=150, sequenceid=235, filesize=30.4 K 2024-11-27T12:27:09,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/.tmp/B/88bdec78876e48c79190c8c2d6ceb92e as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/B/88bdec78876e48c79190c8c2d6ceb92e 2024-11-27T12:27:09,810 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/B/88bdec78876e48c79190c8c2d6ceb92e, entries=150, sequenceid=235, filesize=11.9 K 2024-11-27T12:27:09,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/.tmp/C/8bab4586020d4d5984d6bb534d889caa as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/C/8bab4586020d4d5984d6bb534d889caa 2024-11-27T12:27:09,813 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/C/8bab4586020d4d5984d6bb534d889caa, entries=150, sequenceid=235, filesize=11.9 K 2024-11-27T12:27:09,814 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for a2c58e0d0c97538318beef21ebb34095 in 1660ms, sequenceid=235, compaction requested=false 2024-11-27T12:27:09,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2603): Flush status journal for a2c58e0d0c97538318beef21ebb34095: 2024-11-27T12:27:09,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095. 2024-11-27T12:27:09,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=165 2024-11-27T12:27:09,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36215 {}] master.HMaster(4169): Remote procedure done, pid=165 2024-11-27T12:27:09,817 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=165, resume processing ppid=164 2024-11-27T12:27:09,817 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=165, ppid=164, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.5750 sec 2024-11-27T12:27:09,818 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=164, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees in 2.5790 sec 2024-11-27T12:27:10,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(8855): Flush requested on a2c58e0d0c97538318beef21ebb34095 2024-11-27T12:27:10,131 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T12:27:10,131 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a2c58e0d0c97538318beef21ebb34095 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-27T12:27:10,136 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411271605ba86542c459aaa6089464035a839_a2c58e0d0c97538318beef21ebb34095 is 50, key is test_row_0/A:col10/1732710428990/Put/seqid=0 2024-11-27T12:27:10,149 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:27:10,149 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742408_1584 (size=14894) 2024-11-27T12:27:10,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35506 deadline: 1732710490145, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:27:10,150 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:27:10,150 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:27:10,150 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:27:10,150 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:27:10,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35496 deadline: 1732710490146, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:27:10,150 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:27:10,150 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:27:10,150 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:27:10,153 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:27:10,153 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:27:10,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35492 deadline: 1732710490148, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:27:10,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35442 deadline: 1732710490149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:27:10,154 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:27:10,154 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:27:10,154 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:27:10,154 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:27:10,154 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:27:10,154 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:27:10,259 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:27:10,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35506 deadline: 1732710490257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:27:10,260 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:27:10,260 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:27:10,260 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:27:10,260 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:27:10,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35496 deadline: 1732710490258, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:27:10,260 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:27:10,260 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:27:10,260 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:27:10,260 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:27:10,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35492 deadline: 1732710490258, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:27:10,260 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:27:10,260 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:27:10,260 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:27:10,260 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:27:10,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35442 deadline: 1732710490258, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:27:10,261 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:27:10,261 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:27:10,261 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:27:10,469 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:27:10,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35506 deadline: 1732710490467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:27:10,469 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:27:10,469 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:27:10,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35496 deadline: 1732710490468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:27:10,469 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:27:10,469 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:27:10,470 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:27:10,469 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:27:10,470 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:27:10,470 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:27:10,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35492 deadline: 1732710490468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:27:10,470 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:27:10,470 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:27:10,470 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:27:10,470 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:27:10,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35442 deadline: 1732710490468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:27:10,470 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:27:10,470 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:27:10,470 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:27:10,550 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:27:10,553 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411271605ba86542c459aaa6089464035a839_a2c58e0d0c97538318beef21ebb34095 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411271605ba86542c459aaa6089464035a839_a2c58e0d0c97538318beef21ebb34095 2024-11-27T12:27:10,554 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/.tmp/A/3bf6c4cd22c3406dab36364faaeb7703, store: [table=TestAcidGuarantees family=A region=a2c58e0d0c97538318beef21ebb34095] 2024-11-27T12:27:10,554 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/.tmp/A/3bf6c4cd22c3406dab36364faaeb7703 is 175, key is test_row_0/A:col10/1732710428990/Put/seqid=0 2024-11-27T12:27:10,558 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742409_1585 (size=39849) 2024-11-27T12:27:10,780 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:27:10,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35506 deadline: 1732710490778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:27:10,781 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:27:10,781 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:27:10,781 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:27:10,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35496 deadline: 1732710490778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:27:10,781 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:27:10,781 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:27:10,781 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:27:10,781 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:27:10,781 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:27:10,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35492 deadline: 1732710490778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:27:10,781 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:27:10,781 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:27:10,781 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:27:10,781 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:27:10,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35442 deadline: 1732710490778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:27:10,782 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:27:10,782 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:27:10,782 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:27:10,939 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:27:10,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35468 deadline: 1732710490938, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:27:10,939 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 1ba5af43fd99,36251,1732710241631 for 'test_row_2' in a2c58e0d0c97538318beef21ebb34095 of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18303 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T12:27:10,940 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:27:10,940 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:27:10,940 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:27:10,958 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=260, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/.tmp/A/3bf6c4cd22c3406dab36364faaeb7703 2024-11-27T12:27:10,964 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/.tmp/B/f385111ce4224651948b03094395a174 is 50, key is test_row_0/B:col10/1732710428990/Put/seqid=0 2024-11-27T12:27:10,969 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742410_1586 (size=12251) 2024-11-27T12:27:11,290 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:27:11,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35506 deadline: 1732710491288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:27:11,291 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:27:11,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35496 deadline: 1732710491288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:27:11,291 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:27:11,291 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:27:11,291 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:27:11,291 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:27:11,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35492 deadline: 1732710491288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:27:11,291 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:27:11,291 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:27:11,291 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:27:11,291 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:27:11,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35442 deadline: 1732710491288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:27:11,291 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:27:11,291 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:27:11,291 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:27:11,291 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:27:11,291 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:27:11,291 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:27:11,369 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=260 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/.tmp/B/f385111ce4224651948b03094395a174 2024-11-27T12:27:11,375 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/.tmp/C/1f638e7f924c436f8bfc5a0a54b41eae is 50, key is test_row_0/C:col10/1732710428990/Put/seqid=0 2024-11-27T12:27:11,377 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742411_1587 (size=12251) 2024-11-27T12:27:11,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-27T12:27:11,399 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T12:27:11,399 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36215 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-27T12:27:11,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36215 {}] procedure2.ProcedureExecutor(1139): Stored pid=166, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees 2024-11-27T12:27:11,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-27T12:27:11,400 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=166, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T12:27:11,401 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=166, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T12:27:11,401 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=167, ppid=166, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T12:27:11,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-27T12:27:11,553 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-27T12:27:11,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095. 2024-11-27T12:27:11,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095. as already flushing 2024-11-27T12:27:11,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095. 2024-11-27T12:27:11,553 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => a2c58e0d0c97538318beef21ebb34095, NAME => 'TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:27:11,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => a2c58e0d0c97538318beef21ebb34095, NAME => 'TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:27:11,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36215 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a2c58e0d0c97538318beef21ebb34095, NAME => 'TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a2c58e0d0c97538318beef21ebb34095, NAME => 'TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:27:11,705 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-27T12:27:11,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095. 2024-11-27T12:27:11,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095. as already flushing 2024-11-27T12:27:11,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095. 2024-11-27T12:27:11,706 ERROR [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => a2c58e0d0c97538318beef21ebb34095, NAME => 'TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:27:11,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => a2c58e0d0c97538318beef21ebb34095, NAME => 'TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:27:11,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36215 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a2c58e0d0c97538318beef21ebb34095, NAME => 'TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a2c58e0d0c97538318beef21ebb34095, NAME => 'TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T12:27:11,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-27T12:27:11,778 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=260 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/.tmp/C/1f638e7f924c436f8bfc5a0a54b41eae 2024-11-27T12:27:11,781 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/.tmp/A/3bf6c4cd22c3406dab36364faaeb7703 as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/A/3bf6c4cd22c3406dab36364faaeb7703 2024-11-27T12:27:11,784 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/A/3bf6c4cd22c3406dab36364faaeb7703, entries=200, sequenceid=260, filesize=38.9 K 2024-11-27T12:27:11,784 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/.tmp/B/f385111ce4224651948b03094395a174 as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/B/f385111ce4224651948b03094395a174 2024-11-27T12:27:11,787 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/B/f385111ce4224651948b03094395a174, entries=150, sequenceid=260, filesize=12.0 K 2024-11-27T12:27:11,787 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/.tmp/C/1f638e7f924c436f8bfc5a0a54b41eae as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/C/1f638e7f924c436f8bfc5a0a54b41eae 2024-11-27T12:27:11,790 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/C/1f638e7f924c436f8bfc5a0a54b41eae, entries=150, sequenceid=260, filesize=12.0 K 2024-11-27T12:27:11,791 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for a2c58e0d0c97538318beef21ebb34095 in 1660ms, sequenceid=260, compaction requested=true 2024-11-27T12:27:11,791 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a2c58e0d0c97538318beef21ebb34095: 2024-11-27T12:27:11,791 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a2c58e0d0c97538318beef21ebb34095:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T12:27:11,791 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T12:27:11,791 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T12:27:11,791 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a2c58e0d0c97538318beef21ebb34095:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T12:27:11,791 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T12:27:11,791 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a2c58e0d0c97538318beef21ebb34095:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T12:27:11,791 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T12:27:11,791 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T12:27:11,792 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102537 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T12:27:11,792 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HStore(1541): a2c58e0d0c97538318beef21ebb34095/A is initiating minor compaction (all files) 2024-11-27T12:27:11,792 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a2c58e0d0c97538318beef21ebb34095/A in TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095. 2024-11-27T12:27:11,792 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/A/32aa342608ca41c3ae55c681a24811af, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/A/0cba606e06304f04bcba15f05dedb95f, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/A/3bf6c4cd22c3406dab36364faaeb7703] into tmpdir=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/.tmp, totalSize=100.1 K 2024-11-27T12:27:11,792 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095. 2024-11-27T12:27:11,792 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095. files: [hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/A/32aa342608ca41c3ae55c681a24811af, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/A/0cba606e06304f04bcba15f05dedb95f, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/A/3bf6c4cd22c3406dab36364faaeb7703] 2024-11-27T12:27:11,792 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37031 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T12:27:11,792 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] compactions.Compactor(225): Compacting 32aa342608ca41c3ae55c681a24811af, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=220, earliestPutTs=1732710426810 2024-11-27T12:27:11,792 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HStore(1541): a2c58e0d0c97538318beef21ebb34095/B is initiating minor compaction (all files) 2024-11-27T12:27:11,792 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a2c58e0d0c97538318beef21ebb34095/B in TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095. 2024-11-27T12:27:11,792 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/B/00180eca120a42c584a9ef9d5dabed0f, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/B/88bdec78876e48c79190c8c2d6ceb92e, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/B/f385111ce4224651948b03094395a174] into tmpdir=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/.tmp, totalSize=36.2 K 2024-11-27T12:27:11,792 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0cba606e06304f04bcba15f05dedb95f, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1732710426812 2024-11-27T12:27:11,793 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3bf6c4cd22c3406dab36364faaeb7703, keycount=200, bloomtype=ROW, size=38.9 K, encoding=NONE, compression=NONE, seqNum=260, earliestPutTs=1732710428990 2024-11-27T12:27:11,793 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.Compactor(225): Compacting 00180eca120a42c584a9ef9d5dabed0f, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=220, earliestPutTs=1732710426810 2024-11-27T12:27:11,793 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.Compactor(225): Compacting 88bdec78876e48c79190c8c2d6ceb92e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1732710426812 2024-11-27T12:27:11,793 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.Compactor(225): Compacting f385111ce4224651948b03094395a174, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=260, earliestPutTs=1732710428990 2024-11-27T12:27:11,796 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=a2c58e0d0c97538318beef21ebb34095] 2024-11-27T12:27:11,798 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a2c58e0d0c97538318beef21ebb34095#B#compaction#488 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-27T12:27:11,798 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/.tmp/B/285d3e9acf9d40fe85482aebd90592bc is 50, key is test_row_0/B:col10/1732710428990/Put/seqid=0 2024-11-27T12:27:11,799 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411275223699de7084098af82f21d49875cf7_a2c58e0d0c97538318beef21ebb34095 store=[table=TestAcidGuarantees family=A region=a2c58e0d0c97538318beef21ebb34095] 2024-11-27T12:27:11,801 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742412_1588 (size=12831) 2024-11-27T12:27:11,801 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411275223699de7084098af82f21d49875cf7_a2c58e0d0c97538318beef21ebb34095, store=[table=TestAcidGuarantees family=A region=a2c58e0d0c97538318beef21ebb34095] 2024-11-27T12:27:11,802 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411275223699de7084098af82f21d49875cf7_a2c58e0d0c97538318beef21ebb34095 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=a2c58e0d0c97538318beef21ebb34095] 2024-11-27T12:27:11,805 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742413_1589 (size=4469) 2024-11-27T12:27:11,806 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a2c58e0d0c97538318beef21ebb34095#A#compaction#489 average throughput is 2.71 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T12:27:11,806 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/.tmp/A/d4611bd85f86483c97f08bc67d90d95a is 175, key is test_row_0/A:col10/1732710428990/Put/seqid=0 2024-11-27T12:27:11,814 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742414_1590 (size=31785) 2024-11-27T12:27:11,819 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/.tmp/A/d4611bd85f86483c97f08bc67d90d95a as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/A/d4611bd85f86483c97f08bc67d90d95a 2024-11-27T12:27:11,822 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a2c58e0d0c97538318beef21ebb34095/A of a2c58e0d0c97538318beef21ebb34095 into d4611bd85f86483c97f08bc67d90d95a(size=31.0 K), total size for store is 31.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T12:27:11,823 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a2c58e0d0c97538318beef21ebb34095: 2024-11-27T12:27:11,823 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., storeName=a2c58e0d0c97538318beef21ebb34095/A, priority=13, startTime=1732710431791; duration=0sec 2024-11-27T12:27:11,823 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T12:27:11,823 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a2c58e0d0c97538318beef21ebb34095:A 2024-11-27T12:27:11,823 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T12:27:11,823 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37031 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T12:27:11,824 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HStore(1541): a2c58e0d0c97538318beef21ebb34095/C is initiating minor compaction (all files) 2024-11-27T12:27:11,824 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a2c58e0d0c97538318beef21ebb34095/C in TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095. 2024-11-27T12:27:11,824 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/C/962bf29d5ff04658bab0af426812e61b, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/C/8bab4586020d4d5984d6bb534d889caa, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/C/1f638e7f924c436f8bfc5a0a54b41eae] into tmpdir=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/.tmp, totalSize=36.2 K 2024-11-27T12:27:11,824 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] compactions.Compactor(225): Compacting 962bf29d5ff04658bab0af426812e61b, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=220, earliestPutTs=1732710426810 2024-11-27T12:27:11,824 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8bab4586020d4d5984d6bb534d889caa, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1732710426812 2024-11-27T12:27:11,824 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1f638e7f924c436f8bfc5a0a54b41eae, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=260, earliestPutTs=1732710428990 2024-11-27T12:27:11,830 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a2c58e0d0c97538318beef21ebb34095#C#compaction#490 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T12:27:11,831 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/.tmp/C/c28f2d6b092543e9a056079cbf6e8688 is 50, key is test_row_0/C:col10/1732710428990/Put/seqid=0 2024-11-27T12:27:11,834 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742415_1591 (size=12831) 2024-11-27T12:27:11,858 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-27T12:27:11,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095. 2024-11-27T12:27:11,858 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2902): Flushing a2c58e0d0c97538318beef21ebb34095 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-27T12:27:11,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127129f6806ca0d4f3d838409ec1c2c84aa_a2c58e0d0c97538318beef21ebb34095 is 50, key is test_row_0/A:col10/1732710430148/Put/seqid=0 2024-11-27T12:27:11,870 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742416_1592 (size=12454) 2024-11-27T12:27:11,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:27:11,875 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127129f6806ca0d4f3d838409ec1c2c84aa_a2c58e0d0c97538318beef21ebb34095 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127129f6806ca0d4f3d838409ec1c2c84aa_a2c58e0d0c97538318beef21ebb34095 2024-11-27T12:27:11,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/.tmp/A/95985a4571b24ec2aee5aab94c708fa6, store: [table=TestAcidGuarantees family=A region=a2c58e0d0c97538318beef21ebb34095] 2024-11-27T12:27:11,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/.tmp/A/95985a4571b24ec2aee5aab94c708fa6 is 175, key is test_row_0/A:col10/1732710430148/Put/seqid=0 2024-11-27T12:27:11,879 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742417_1593 (size=31255) 2024-11-27T12:27:12,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-27T12:27:12,205 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/.tmp/B/285d3e9acf9d40fe85482aebd90592bc as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/B/285d3e9acf9d40fe85482aebd90592bc 2024-11-27T12:27:12,209 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a2c58e0d0c97538318beef21ebb34095/B of a2c58e0d0c97538318beef21ebb34095 into 285d3e9acf9d40fe85482aebd90592bc(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T12:27:12,209 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a2c58e0d0c97538318beef21ebb34095: 2024-11-27T12:27:12,209 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., storeName=a2c58e0d0c97538318beef21ebb34095/B, priority=13, startTime=1732710431791; duration=0sec 2024-11-27T12:27:12,209 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T12:27:12,209 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a2c58e0d0c97538318beef21ebb34095:B 2024-11-27T12:27:12,238 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/.tmp/C/c28f2d6b092543e9a056079cbf6e8688 as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/C/c28f2d6b092543e9a056079cbf6e8688 2024-11-27T12:27:12,243 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a2c58e0d0c97538318beef21ebb34095/C of a2c58e0d0c97538318beef21ebb34095 into c28f2d6b092543e9a056079cbf6e8688(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T12:27:12,243 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a2c58e0d0c97538318beef21ebb34095: 2024-11-27T12:27:12,243 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., storeName=a2c58e0d0c97538318beef21ebb34095/C, priority=13, startTime=1732710431791; duration=0sec 2024-11-27T12:27:12,243 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T12:27:12,243 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a2c58e0d0c97538318beef21ebb34095:C 2024-11-27T12:27:12,280 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=275, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/.tmp/A/95985a4571b24ec2aee5aab94c708fa6 2024-11-27T12:27:12,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/.tmp/B/6dba99719e6040cea7926a438bd87862 is 50, key is test_row_0/B:col10/1732710430148/Put/seqid=0 2024-11-27T12:27:12,291 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742418_1594 (size=12301) 2024-11-27T12:27:12,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(8855): Flush requested on a2c58e0d0c97538318beef21ebb34095 2024-11-27T12:27:12,300 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095. as already flushing 2024-11-27T12:27:12,330 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:27:12,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35506 deadline: 1732710492325, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:27:12,331 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:27:12,331 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:27:12,331 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:27:12,334 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:27:12,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35492 deadline: 1732710492329, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:27:12,334 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:27:12,334 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:27:12,334 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:27:12,335 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:27:12,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35442 deadline: 1732710492330, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:27:12,335 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:27:12,335 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:27:12,335 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:27:12,335 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:27:12,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35496 deadline: 1732710492330, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:27:12,336 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:27:12,336 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:27:12,336 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:27:12,440 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:27:12,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35506 deadline: 1732710492437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:27:12,441 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:27:12,441 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:27:12,441 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:27:12,441 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:27:12,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35492 deadline: 1732710492437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:27:12,441 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:27:12,441 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:27:12,441 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:27:12,441 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:27:12,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35442 deadline: 1732710492438, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:27:12,442 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:27:12,442 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:27:12,442 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:27:12,442 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:27:12,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35496 deadline: 1732710492438, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:27:12,442 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:27:12,442 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:27:12,442 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:27:12,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-27T12:27:12,579 INFO [Thread-2347 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2347. 2024-11-27T12:27:12,579 DEBUG [Thread-2347 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T12:27:12,579 DEBUG [Thread-2347 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T12:27:12,579 DEBUG [Thread-2347 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T12:27:12,579 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T12:27:12,581 INFO [Thread-2341 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2341. 2024-11-27T12:27:12,581 DEBUG [Thread-2341 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T12:27:12,581 DEBUG [Thread-2341 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T12:27:12,581 DEBUG [Thread-2341 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T12:27:12,581 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T12:27:12,581 INFO [Thread-2343 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2343. 2024-11-27T12:27:12,582 DEBUG [Thread-2343 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T12:27:12,582 DEBUG [Thread-2343 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T12:27:12,582 DEBUG [Thread-2343 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T12:27:12,582 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T12:27:12,582 INFO [Thread-2339 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2339. 2024-11-27T12:27:12,583 DEBUG [Thread-2339 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T12:27:12,583 DEBUG [Thread-2339 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T12:27:12,583 DEBUG [Thread-2339 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T12:27:12,583 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T12:27:12,584 INFO [Thread-2345 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2345. 2024-11-27T12:27:12,584 DEBUG [Thread-2345 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T12:27:12,584 DEBUG [Thread-2345 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T12:27:12,584 DEBUG [Thread-2345 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T12:27:12,584 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T12:27:12,648 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:27:12,648 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:27:12,648 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:27:12,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35506 deadline: 1732710492648, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:27:12,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35492 deadline: 1732710492648, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:27:12,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35442 deadline: 1732710492648, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:27:12,648 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:27:12,648 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:27:12,648 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:27:12,648 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:27:12,648 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:27:12,648 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:27:12,648 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:27:12,648 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:27:12,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35496 deadline: 1732710492648, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:27:12,648 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:27:12,648 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:27:12,649 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:27:12,649 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:27:12,649 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:27:12,691 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=275 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/.tmp/B/6dba99719e6040cea7926a438bd87862 2024-11-27T12:27:12,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/.tmp/C/e48133d50c3944ad88005edde347376e is 50, key is test_row_0/C:col10/1732710430148/Put/seqid=0 2024-11-27T12:27:12,703 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742419_1595 (size=12301) 2024-11-27T12:27:12,959 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:27:12,959 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:27:12,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35442 deadline: 1732710492958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:27:12,959 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:27:12,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35492 deadline: 1732710492958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:27:12,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35506 deadline: 1732710492959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:27:12,960 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:27:12,960 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T12:27:12,960 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:27:12,960 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:27:12,960 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:27:12,960 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:27:12,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35496 deadline: 1732710492959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 2024-11-27T12:27:12,960 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:27:12,960 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:27:12,960 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:27:12,960 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:27:12,961 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 , the old value is region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:27:12,961 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T12:27:12,961 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., hostname=1ba5af43fd99,36251,1732710241631, seqNum=5 because the exception is null or not the one we care about 2024-11-27T12:27:13,105 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=275 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/.tmp/C/e48133d50c3944ad88005edde347376e 2024-11-27T12:27:13,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/.tmp/A/95985a4571b24ec2aee5aab94c708fa6 as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/A/95985a4571b24ec2aee5aab94c708fa6 2024-11-27T12:27:13,119 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/A/95985a4571b24ec2aee5aab94c708fa6, entries=150, sequenceid=275, filesize=30.5 K 2024-11-27T12:27:13,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/.tmp/B/6dba99719e6040cea7926a438bd87862 as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/B/6dba99719e6040cea7926a438bd87862 2024-11-27T12:27:13,123 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/B/6dba99719e6040cea7926a438bd87862, entries=150, sequenceid=275, filesize=12.0 K 2024-11-27T12:27:13,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/.tmp/C/e48133d50c3944ad88005edde347376e as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/C/e48133d50c3944ad88005edde347376e 2024-11-27T12:27:13,127 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/C/e48133d50c3944ad88005edde347376e, entries=150, sequenceid=275, filesize=12.0 K 2024-11-27T12:27:13,128 INFO [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=134.18 KB/137400 for a2c58e0d0c97538318beef21ebb34095 in 1270ms, sequenceid=275, compaction requested=false 2024-11-27T12:27:13,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2603): Flush status journal for a2c58e0d0c97538318beef21ebb34095: 2024-11-27T12:27:13,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095. 2024-11-27T12:27:13,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1ba5af43fd99:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=167 2024-11-27T12:27:13,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36215 {}] master.HMaster(4169): Remote procedure done, pid=167 2024-11-27T12:27:13,132 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=167, resume processing ppid=166 2024-11-27T12:27:13,132 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=167, ppid=166, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7290 sec 2024-11-27T12:27:13,134 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=166, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees in 1.7340 sec 2024-11-27T12:27:13,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36251 {}] regionserver.HRegion(8855): Flush requested on a2c58e0d0c97538318beef21ebb34095 2024-11-27T12:27:13,474 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T12:27:13,474 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a2c58e0d0c97538318beef21ebb34095 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-27T12:27:13,475 INFO [Thread-2330 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2330. 2024-11-27T12:27:13,475 INFO [Thread-2336 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2336. 2024-11-27T12:27:13,475 DEBUG [Thread-2336 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T12:27:13,475 DEBUG [Thread-2330 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T12:27:13,475 DEBUG [Thread-2336 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T12:27:13,475 DEBUG [Thread-2330 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T12:27:13,475 DEBUG [Thread-2336 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T12:27:13,475 DEBUG [Thread-2330 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T12:27:13,475 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T12:27:13,475 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T12:27:13,475 INFO [Thread-2332 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2332. 2024-11-27T12:27:13,475 DEBUG [Thread-2332 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T12:27:13,475 DEBUG [Thread-2332 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T12:27:13,476 DEBUG [Thread-2332 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T12:27:13,476 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T12:27:13,477 INFO [Thread-2328 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2328. 2024-11-27T12:27:13,477 DEBUG [Thread-2328 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T12:27:13,477 DEBUG [Thread-2328 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T12:27:13,477 DEBUG [Thread-2328 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T12:27:13,477 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T12:27:13,482 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112738f71963cbfe469ab69da02c97c2e8e7_a2c58e0d0c97538318beef21ebb34095 is 50, key is test_row_0/A:col10/1732710433469/Put/seqid=0 2024-11-27T12:27:13,485 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742420_1596 (size=12454) 2024-11-27T12:27:13,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-27T12:27:13,549 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T12:27:13,887 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:27:13,895 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112738f71963cbfe469ab69da02c97c2e8e7_a2c58e0d0c97538318beef21ebb34095 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112738f71963cbfe469ab69da02c97c2e8e7_a2c58e0d0c97538318beef21ebb34095 2024-11-27T12:27:13,896 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/.tmp/A/3a6d99f44c204a74814d1a6776655587, store: [table=TestAcidGuarantees family=A region=a2c58e0d0c97538318beef21ebb34095] 2024-11-27T12:27:13,896 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/.tmp/A/3a6d99f44c204a74814d1a6776655587 is 175, key is test_row_0/A:col10/1732710433469/Put/seqid=0 2024-11-27T12:27:13,900 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742421_1597 (size=31255) 2024-11-27T12:27:14,302 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=303, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/.tmp/A/3a6d99f44c204a74814d1a6776655587 2024-11-27T12:27:14,315 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/.tmp/B/d5888bdca7e44e228c8a4de310e008fb is 50, key is test_row_0/B:col10/1732710433469/Put/seqid=0 2024-11-27T12:27:14,318 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742422_1598 (size=12301) 2024-11-27T12:27:14,720 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=303 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/.tmp/B/d5888bdca7e44e228c8a4de310e008fb 2024-11-27T12:27:14,775 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/.tmp/C/48df09131d674cb8b4d94e260d35c9ea is 50, key is test_row_0/C:col10/1732710433469/Put/seqid=0 2024-11-27T12:27:14,778 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742423_1599 (size=12301) 2024-11-27T12:27:15,180 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=303 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/.tmp/C/48df09131d674cb8b4d94e260d35c9ea 2024-11-27T12:27:15,189 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/.tmp/A/3a6d99f44c204a74814d1a6776655587 as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/A/3a6d99f44c204a74814d1a6776655587 2024-11-27T12:27:15,195 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/A/3a6d99f44c204a74814d1a6776655587, entries=150, sequenceid=303, filesize=30.5 K 2024-11-27T12:27:15,196 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/.tmp/B/d5888bdca7e44e228c8a4de310e008fb as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/B/d5888bdca7e44e228c8a4de310e008fb 2024-11-27T12:27:15,199 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/B/d5888bdca7e44e228c8a4de310e008fb, entries=150, sequenceid=303, filesize=12.0 K 2024-11-27T12:27:15,200 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/.tmp/C/48df09131d674cb8b4d94e260d35c9ea as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/C/48df09131d674cb8b4d94e260d35c9ea 2024-11-27T12:27:15,203 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/C/48df09131d674cb8b4d94e260d35c9ea, entries=150, sequenceid=303, filesize=12.0 K 2024-11-27T12:27:15,204 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=6.71 KB/6870 for a2c58e0d0c97538318beef21ebb34095 in 1730ms, sequenceid=303, compaction requested=true 2024-11-27T12:27:15,204 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a2c58e0d0c97538318beef21ebb34095: 2024-11-27T12:27:15,204 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a2c58e0d0c97538318beef21ebb34095:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T12:27:15,204 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T12:27:15,204 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T12:27:15,204 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a2c58e0d0c97538318beef21ebb34095:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T12:27:15,204 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T12:27:15,204 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a2c58e0d0c97538318beef21ebb34095:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T12:27:15,204 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T12:27:15,204 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T12:27:15,205 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94295 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T12:27:15,205 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37433 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T12:27:15,205 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HStore(1541): a2c58e0d0c97538318beef21ebb34095/A is initiating minor compaction (all files) 2024-11-27T12:27:15,205 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HStore(1541): a2c58e0d0c97538318beef21ebb34095/B is initiating minor compaction (all files) 2024-11-27T12:27:15,205 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a2c58e0d0c97538318beef21ebb34095/A in TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095. 2024-11-27T12:27:15,205 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a2c58e0d0c97538318beef21ebb34095/B in TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095. 2024-11-27T12:27:15,205 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/A/d4611bd85f86483c97f08bc67d90d95a, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/A/95985a4571b24ec2aee5aab94c708fa6, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/A/3a6d99f44c204a74814d1a6776655587] into tmpdir=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/.tmp, totalSize=92.1 K 2024-11-27T12:27:15,205 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/B/285d3e9acf9d40fe85482aebd90592bc, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/B/6dba99719e6040cea7926a438bd87862, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/B/d5888bdca7e44e228c8a4de310e008fb] into tmpdir=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/.tmp, totalSize=36.6 K 2024-11-27T12:27:15,205 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095. 2024-11-27T12:27:15,205 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095. files: [hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/A/d4611bd85f86483c97f08bc67d90d95a, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/A/95985a4571b24ec2aee5aab94c708fa6, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/A/3a6d99f44c204a74814d1a6776655587] 2024-11-27T12:27:15,205 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] compactions.Compactor(225): Compacting 285d3e9acf9d40fe85482aebd90592bc, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=260, earliestPutTs=1732710428990 2024-11-27T12:27:15,206 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.Compactor(225): Compacting d4611bd85f86483c97f08bc67d90d95a, keycount=150, bloomtype=ROW, size=31.0 K, encoding=NONE, compression=NONE, seqNum=260, earliestPutTs=1732710428990 2024-11-27T12:27:15,206 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6dba99719e6040cea7926a438bd87862, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1732710430144 2024-11-27T12:27:15,206 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.Compactor(225): Compacting 95985a4571b24ec2aee5aab94c708fa6, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1732710430144 2024-11-27T12:27:15,206 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] compactions.Compactor(225): Compacting d5888bdca7e44e228c8a4de310e008fb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=303, earliestPutTs=1732710433469 2024-11-27T12:27:15,206 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] compactions.Compactor(225): Compacting 3a6d99f44c204a74814d1a6776655587, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=303, earliestPutTs=1732710433469 2024-11-27T12:27:15,211 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=a2c58e0d0c97538318beef21ebb34095] 2024-11-27T12:27:15,212 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a2c58e0d0c97538318beef21ebb34095#B#compaction#497 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-27T12:27:15,213 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241127bbcaeb65ff564432b56d0eff4487c9dc_a2c58e0d0c97538318beef21ebb34095 store=[table=TestAcidGuarantees family=A region=a2c58e0d0c97538318beef21ebb34095] 2024-11-27T12:27:15,213 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/.tmp/B/8e1685d17568489593bc12c7c49d9b96 is 50, key is test_row_0/B:col10/1732710433469/Put/seqid=0 2024-11-27T12:27:15,215 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241127bbcaeb65ff564432b56d0eff4487c9dc_a2c58e0d0c97538318beef21ebb34095, store=[table=TestAcidGuarantees family=A region=a2c58e0d0c97538318beef21ebb34095] 2024-11-27T12:27:15,215 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127bbcaeb65ff564432b56d0eff4487c9dc_a2c58e0d0c97538318beef21ebb34095 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=a2c58e0d0c97538318beef21ebb34095] 2024-11-27T12:27:15,218 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742425_1601 (size=4469) 2024-11-27T12:27:15,219 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742424_1600 (size=12983) 2024-11-27T12:27:15,620 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a2c58e0d0c97538318beef21ebb34095#A#compaction#498 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T12:27:15,621 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/.tmp/A/8af8aca92f844976aae7072a306dae45 is 175, key is test_row_0/A:col10/1732710433469/Put/seqid=0 2024-11-27T12:27:15,627 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742426_1602 (size=31937) 2024-11-27T12:27:15,628 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/.tmp/B/8e1685d17568489593bc12c7c49d9b96 as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/B/8e1685d17568489593bc12c7c49d9b96 2024-11-27T12:27:15,632 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a2c58e0d0c97538318beef21ebb34095/B of a2c58e0d0c97538318beef21ebb34095 into 8e1685d17568489593bc12c7c49d9b96(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T12:27:15,632 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a2c58e0d0c97538318beef21ebb34095: 2024-11-27T12:27:15,632 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., storeName=a2c58e0d0c97538318beef21ebb34095/B, priority=13, startTime=1732710435204; duration=0sec 2024-11-27T12:27:15,632 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T12:27:15,632 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a2c58e0d0c97538318beef21ebb34095:B 2024-11-27T12:27:15,632 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T12:27:15,633 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37433 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T12:27:15,633 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HStore(1541): a2c58e0d0c97538318beef21ebb34095/C is initiating minor compaction (all files) 2024-11-27T12:27:15,633 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a2c58e0d0c97538318beef21ebb34095/C in TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095. 2024-11-27T12:27:15,633 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/C/c28f2d6b092543e9a056079cbf6e8688, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/C/e48133d50c3944ad88005edde347376e, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/C/48df09131d674cb8b4d94e260d35c9ea] into tmpdir=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/.tmp, totalSize=36.6 K 2024-11-27T12:27:15,633 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] compactions.Compactor(225): Compacting c28f2d6b092543e9a056079cbf6e8688, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=260, earliestPutTs=1732710428990 2024-11-27T12:27:15,633 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] compactions.Compactor(225): Compacting e48133d50c3944ad88005edde347376e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1732710430144 2024-11-27T12:27:15,634 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] compactions.Compactor(225): Compacting 48df09131d674cb8b4d94e260d35c9ea, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=303, earliestPutTs=1732710433469 2024-11-27T12:27:15,639 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a2c58e0d0c97538318beef21ebb34095#C#compaction#499 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T12:27:15,639 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/.tmp/C/ce930e4d820e44c7a5a03dea420f9f0b is 50, key is test_row_0/C:col10/1732710433469/Put/seqid=0 2024-11-27T12:27:15,642 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742427_1603 (size=12983) 2024-11-27T12:27:16,034 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/.tmp/A/8af8aca92f844976aae7072a306dae45 as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/A/8af8aca92f844976aae7072a306dae45 2024-11-27T12:27:16,039 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a2c58e0d0c97538318beef21ebb34095/A of a2c58e0d0c97538318beef21ebb34095 into 8af8aca92f844976aae7072a306dae45(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T12:27:16,039 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a2c58e0d0c97538318beef21ebb34095: 2024-11-27T12:27:16,039 INFO [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., storeName=a2c58e0d0c97538318beef21ebb34095/A, priority=13, startTime=1732710435204; duration=0sec 2024-11-27T12:27:16,039 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T12:27:16,039 DEBUG [RS:0;1ba5af43fd99:36251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a2c58e0d0c97538318beef21ebb34095:A 2024-11-27T12:27:16,048 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/.tmp/C/ce930e4d820e44c7a5a03dea420f9f0b as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/C/ce930e4d820e44c7a5a03dea420f9f0b 2024-11-27T12:27:16,054 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a2c58e0d0c97538318beef21ebb34095/C of a2c58e0d0c97538318beef21ebb34095 into ce930e4d820e44c7a5a03dea420f9f0b(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T12:27:16,054 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a2c58e0d0c97538318beef21ebb34095: 2024-11-27T12:27:16,054 INFO [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., storeName=a2c58e0d0c97538318beef21ebb34095/C, priority=13, startTime=1732710435204; duration=0sec 2024-11-27T12:27:16,054 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T12:27:16,054 DEBUG [RS:0;1ba5af43fd99:36251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a2c58e0d0c97538318beef21ebb34095:C 2024-11-27T12:27:20,981 INFO [Thread-2334 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2334. 2024-11-27T12:27:20,982 DEBUG [Thread-2334 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T12:27:20,982 DEBUG [Thread-2334 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T12:27:20,982 DEBUG [Thread-2334 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T12:27:20,982 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T12:27:20,983 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-27T12:27:20,983 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 63 2024-11-27T12:27:20,983 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 62 2024-11-27T12:27:20,983 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 55 2024-11-27T12:27:20,983 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 7 2024-11-27T12:27:20,983 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 54 2024-11-27T12:27:20,983 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-27T12:27:20,983 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4126 2024-11-27T12:27:20,983 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4007 2024-11-27T12:27:20,983 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4183 2024-11-27T12:27:20,983 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4138 2024-11-27T12:27:20,983 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4013 2024-11-27T12:27:20,983 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-27T12:27:20,983 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-27T12:27:20,983 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobGetAtomicity(AcidGuaranteesTestBase.java:122) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-27T12:27:20,984 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T12:27:20,984 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T12:27:20,984 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T12:27:20,985 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36215 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-11-27T12:27:20,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36215 {}] procedure2.ProcedureExecutor(1139): Stored pid=168, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-27T12:27:20,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-27T12:27:20,992 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732710440991"}]},"ts":"1732710440991"} 2024-11-27T12:27:20,993 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-27T12:27:20,993 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-27T12:27:20,994 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=169, ppid=168, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-27T12:27:20,996 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=170, ppid=169, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a2c58e0d0c97538318beef21ebb34095, UNASSIGN}] 2024-11-27T12:27:20,996 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=170, ppid=169, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a2c58e0d0c97538318beef21ebb34095, UNASSIGN 2024-11-27T12:27:20,997 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=170 updating hbase:meta row=a2c58e0d0c97538318beef21ebb34095, regionState=CLOSING, regionLocation=1ba5af43fd99,36251,1732710241631 2024-11-27T12:27:20,998 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=170, ppid=169, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=a2c58e0d0c97538318beef21ebb34095, UNASSIGN because future has completed 2024-11-27T12:27:20,999 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-27T12:27:20,999 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=171, ppid=170, state=RUNNABLE, hasLock=false; CloseRegionProcedure a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631}] 2024-11-27T12:27:21,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-27T12:27:21,153 INFO [RS_CLOSE_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] handler.UnassignRegionHandler(122): Close a2c58e0d0c97538318beef21ebb34095 2024-11-27T12:27:21,153 DEBUG [RS_CLOSE_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-27T12:27:21,153 DEBUG [RS_CLOSE_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HRegion(1722): Closing a2c58e0d0c97538318beef21ebb34095, disabling compactions & flushes 2024-11-27T12:27:21,153 INFO [RS_CLOSE_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095. 2024-11-27T12:27:21,153 DEBUG [RS_CLOSE_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095. 2024-11-27T12:27:21,153 DEBUG [RS_CLOSE_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095. after waiting 0 ms 2024-11-27T12:27:21,153 DEBUG [RS_CLOSE_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095. 2024-11-27T12:27:21,154 INFO [RS_CLOSE_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HRegion(2902): Flushing a2c58e0d0c97538318beef21ebb34095 3/3 column families, dataSize=13.42 KB heapSize=35.91 KB 2024-11-27T12:27:21,165 DEBUG [RS_CLOSE_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127ccade85c6fd24b3abba1726bd51ef502_a2c58e0d0c97538318beef21ebb34095 is 50, key is test_row_0/A:col10/1732710433475/Put/seqid=0 2024-11-27T12:27:21,168 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742428_1604 (size=9914) 2024-11-27T12:27:21,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-27T12:27:21,570 DEBUG [RS_CLOSE_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T12:27:21,579 INFO [RS_CLOSE_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127ccade85c6fd24b3abba1726bd51ef502_a2c58e0d0c97538318beef21ebb34095 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127ccade85c6fd24b3abba1726bd51ef502_a2c58e0d0c97538318beef21ebb34095 2024-11-27T12:27:21,580 DEBUG [RS_CLOSE_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/.tmp/A/83d3111070984a6d9fc782086ffbbb57, store: [table=TestAcidGuarantees family=A region=a2c58e0d0c97538318beef21ebb34095] 2024-11-27T12:27:21,581 DEBUG [RS_CLOSE_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/.tmp/A/83d3111070984a6d9fc782086ffbbb57 is 175, key is test_row_0/A:col10/1732710433475/Put/seqid=0 2024-11-27T12:27:21,584 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742429_1605 (size=22561) 2024-11-27T12:27:21,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-27T12:27:21,986 INFO [RS_CLOSE_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=311, memsize=4.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/.tmp/A/83d3111070984a6d9fc782086ffbbb57 2024-11-27T12:27:22,000 DEBUG [RS_CLOSE_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/.tmp/B/22692424d5404ef4a157e34df92d5a39 is 50, key is test_row_0/B:col10/1732710433475/Put/seqid=0 2024-11-27T12:27:22,003 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742430_1606 (size=9857) 2024-11-27T12:27:22,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-27T12:27:22,405 INFO [RS_CLOSE_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=311 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/.tmp/B/22692424d5404ef4a157e34df92d5a39 2024-11-27T12:27:22,417 DEBUG [RS_CLOSE_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/.tmp/C/532ddce2f8844385b7d1e03f94e0045f is 50, key is test_row_0/C:col10/1732710433475/Put/seqid=0 2024-11-27T12:27:22,420 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742431_1607 (size=9857) 2024-11-27T12:27:22,822 INFO [RS_CLOSE_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=311 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/.tmp/C/532ddce2f8844385b7d1e03f94e0045f 2024-11-27T12:27:22,832 DEBUG [RS_CLOSE_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/.tmp/A/83d3111070984a6d9fc782086ffbbb57 as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/A/83d3111070984a6d9fc782086ffbbb57 2024-11-27T12:27:22,838 INFO [RS_CLOSE_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/A/83d3111070984a6d9fc782086ffbbb57, entries=100, sequenceid=311, filesize=22.0 K 2024-11-27T12:27:22,839 DEBUG [RS_CLOSE_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/.tmp/B/22692424d5404ef4a157e34df92d5a39 as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/B/22692424d5404ef4a157e34df92d5a39 2024-11-27T12:27:22,843 INFO [RS_CLOSE_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/B/22692424d5404ef4a157e34df92d5a39, entries=100, sequenceid=311, filesize=9.6 K 2024-11-27T12:27:22,844 DEBUG [RS_CLOSE_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/.tmp/C/532ddce2f8844385b7d1e03f94e0045f as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/C/532ddce2f8844385b7d1e03f94e0045f 2024-11-27T12:27:22,847 INFO [RS_CLOSE_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/C/532ddce2f8844385b7d1e03f94e0045f, entries=100, sequenceid=311, filesize=9.6 K 2024-11-27T12:27:22,847 INFO [RS_CLOSE_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HRegion(3140): Finished flush of dataSize ~13.42 KB/13740, heapSize ~35.86 KB/36720, currentSize=0 B/0 for a2c58e0d0c97538318beef21ebb34095 in 1694ms, sequenceid=311, compaction requested=false 2024-11-27T12:27:22,848 DEBUG [StoreCloser-TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/A/11a73d08d3b543efbb7bb2183d051530, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/A/f3fca4d358d1462ca7244c4e1d2d482e, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/A/b5469766cfcb45a7bee390bfa229c9bc, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/A/f7abff7285e24386ab04f8d15be53802, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/A/51773740d62b4903bd86cd2de3b40fce, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/A/50ff434510a643108f15bdce75ac1a92, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/A/0663abe7b9644947ba30247d80ae337d, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/A/f5313edae9e74de994a3b363bd20a7e9, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/A/efaada395c22467da7016e016c858ca5, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/A/40eb2171a726414d976f9b21bf892db9, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/A/9c726caa786e4965a39b59b3effb0c33, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/A/687530490fb947ffbc47a64e19e8b627, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/A/5cb5972502034a0baea3f254a5ef1fa0, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/A/78d4650d42b54761a5d81b875b0f7add, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/A/32aa342608ca41c3ae55c681a24811af, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/A/0cba606e06304f04bcba15f05dedb95f, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/A/3bf6c4cd22c3406dab36364faaeb7703, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/A/d4611bd85f86483c97f08bc67d90d95a, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/A/95985a4571b24ec2aee5aab94c708fa6, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/A/3a6d99f44c204a74814d1a6776655587] to archive 2024-11-27T12:27:22,849 DEBUG [StoreCloser-TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-27T12:27:22,850 DEBUG [StoreCloser-TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/A/11a73d08d3b543efbb7bb2183d051530 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/A/11a73d08d3b543efbb7bb2183d051530 2024-11-27T12:27:22,851 DEBUG [StoreCloser-TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/A/f3fca4d358d1462ca7244c4e1d2d482e to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/A/f3fca4d358d1462ca7244c4e1d2d482e 2024-11-27T12:27:22,852 DEBUG [StoreCloser-TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/A/b5469766cfcb45a7bee390bfa229c9bc to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/A/b5469766cfcb45a7bee390bfa229c9bc 2024-11-27T12:27:22,853 DEBUG [StoreCloser-TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/A/f7abff7285e24386ab04f8d15be53802 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/A/f7abff7285e24386ab04f8d15be53802 2024-11-27T12:27:22,854 DEBUG [StoreCloser-TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/A/51773740d62b4903bd86cd2de3b40fce to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/A/51773740d62b4903bd86cd2de3b40fce 2024-11-27T12:27:22,854 DEBUG [StoreCloser-TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/A/50ff434510a643108f15bdce75ac1a92 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/A/50ff434510a643108f15bdce75ac1a92 2024-11-27T12:27:22,855 DEBUG [StoreCloser-TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/A/0663abe7b9644947ba30247d80ae337d to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/A/0663abe7b9644947ba30247d80ae337d 2024-11-27T12:27:22,856 DEBUG [StoreCloser-TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/A/f5313edae9e74de994a3b363bd20a7e9 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/A/f5313edae9e74de994a3b363bd20a7e9 2024-11-27T12:27:22,857 DEBUG [StoreCloser-TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/A/efaada395c22467da7016e016c858ca5 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/A/efaada395c22467da7016e016c858ca5 2024-11-27T12:27:22,858 DEBUG [StoreCloser-TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/A/40eb2171a726414d976f9b21bf892db9 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/A/40eb2171a726414d976f9b21bf892db9 2024-11-27T12:27:22,859 DEBUG [StoreCloser-TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/A/9c726caa786e4965a39b59b3effb0c33 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/A/9c726caa786e4965a39b59b3effb0c33 2024-11-27T12:27:22,860 DEBUG [StoreCloser-TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/A/687530490fb947ffbc47a64e19e8b627 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/A/687530490fb947ffbc47a64e19e8b627 2024-11-27T12:27:22,860 DEBUG [StoreCloser-TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/A/5cb5972502034a0baea3f254a5ef1fa0 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/A/5cb5972502034a0baea3f254a5ef1fa0 2024-11-27T12:27:22,861 DEBUG [StoreCloser-TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/A/78d4650d42b54761a5d81b875b0f7add to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/A/78d4650d42b54761a5d81b875b0f7add 2024-11-27T12:27:22,862 DEBUG [StoreCloser-TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/A/32aa342608ca41c3ae55c681a24811af to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/A/32aa342608ca41c3ae55c681a24811af 2024-11-27T12:27:22,862 DEBUG [StoreCloser-TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/A/0cba606e06304f04bcba15f05dedb95f to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/A/0cba606e06304f04bcba15f05dedb95f 2024-11-27T12:27:22,863 DEBUG [StoreCloser-TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/A/3bf6c4cd22c3406dab36364faaeb7703 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/A/3bf6c4cd22c3406dab36364faaeb7703 2024-11-27T12:27:22,864 DEBUG [StoreCloser-TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/A/d4611bd85f86483c97f08bc67d90d95a to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/A/d4611bd85f86483c97f08bc67d90d95a 2024-11-27T12:27:22,864 DEBUG [StoreCloser-TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/A/95985a4571b24ec2aee5aab94c708fa6 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/A/95985a4571b24ec2aee5aab94c708fa6 2024-11-27T12:27:22,865 DEBUG [StoreCloser-TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/A/3a6d99f44c204a74814d1a6776655587 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/A/3a6d99f44c204a74814d1a6776655587 2024-11-27T12:27:22,866 DEBUG [StoreCloser-TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/B/c74cd445ae004974b3accff81f8d434f, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/B/0fc4365b36d64a0fb7c899060949712a, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/B/7e84549f5d284ac5b86eaee3bc3d57ed, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/B/76bbaef2dc4e4276b7a161b3768bb0aa, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/B/37ecd6560e8c4f4fbae5f12cc4a23042, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/B/15a1546f025a48faa931bc98bb25fc15, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/B/416010e191074cfd839210937a8bae9e, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/B/80cde32d8e3042b7994566a0370754ea, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/B/1eff988f7949427488836acf331b6a14, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/B/7a795df9d9564cc3a94b814db6582ec8, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/B/4401eefbca9842348470a50f6d4fbc51, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/B/116d99be4c994e6b9824db994bbcdef8, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/B/2d9905f5b3c9464394eec2ed540ed6ac, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/B/00180eca120a42c584a9ef9d5dabed0f, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/B/2f83a6476b6942f5aa1def86cf7ec5f3, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/B/88bdec78876e48c79190c8c2d6ceb92e, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/B/285d3e9acf9d40fe85482aebd90592bc, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/B/f385111ce4224651948b03094395a174, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/B/6dba99719e6040cea7926a438bd87862, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/B/d5888bdca7e44e228c8a4de310e008fb] to archive 2024-11-27T12:27:22,867 DEBUG [StoreCloser-TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-27T12:27:22,868 DEBUG [StoreCloser-TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/B/c74cd445ae004974b3accff81f8d434f to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/B/c74cd445ae004974b3accff81f8d434f 2024-11-27T12:27:22,868 DEBUG [StoreCloser-TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/B/0fc4365b36d64a0fb7c899060949712a to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/B/0fc4365b36d64a0fb7c899060949712a 2024-11-27T12:27:22,869 DEBUG [StoreCloser-TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/B/7e84549f5d284ac5b86eaee3bc3d57ed to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/B/7e84549f5d284ac5b86eaee3bc3d57ed 2024-11-27T12:27:22,870 DEBUG [StoreCloser-TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/B/76bbaef2dc4e4276b7a161b3768bb0aa to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/B/76bbaef2dc4e4276b7a161b3768bb0aa 2024-11-27T12:27:22,870 DEBUG [StoreCloser-TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/B/37ecd6560e8c4f4fbae5f12cc4a23042 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/B/37ecd6560e8c4f4fbae5f12cc4a23042 2024-11-27T12:27:22,871 DEBUG [StoreCloser-TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/B/15a1546f025a48faa931bc98bb25fc15 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/B/15a1546f025a48faa931bc98bb25fc15 2024-11-27T12:27:22,872 DEBUG [StoreCloser-TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/B/416010e191074cfd839210937a8bae9e to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/B/416010e191074cfd839210937a8bae9e 2024-11-27T12:27:22,872 DEBUG [StoreCloser-TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/B/80cde32d8e3042b7994566a0370754ea to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/B/80cde32d8e3042b7994566a0370754ea 2024-11-27T12:27:22,873 DEBUG [StoreCloser-TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/B/1eff988f7949427488836acf331b6a14 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/B/1eff988f7949427488836acf331b6a14 2024-11-27T12:27:22,874 DEBUG [StoreCloser-TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/B/7a795df9d9564cc3a94b814db6582ec8 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/B/7a795df9d9564cc3a94b814db6582ec8 2024-11-27T12:27:22,875 DEBUG [StoreCloser-TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/B/4401eefbca9842348470a50f6d4fbc51 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/B/4401eefbca9842348470a50f6d4fbc51 2024-11-27T12:27:22,875 DEBUG [StoreCloser-TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/B/116d99be4c994e6b9824db994bbcdef8 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/B/116d99be4c994e6b9824db994bbcdef8 2024-11-27T12:27:22,876 DEBUG [StoreCloser-TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/B/2d9905f5b3c9464394eec2ed540ed6ac to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/B/2d9905f5b3c9464394eec2ed540ed6ac 2024-11-27T12:27:22,877 DEBUG [StoreCloser-TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/B/00180eca120a42c584a9ef9d5dabed0f to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/B/00180eca120a42c584a9ef9d5dabed0f 2024-11-27T12:27:22,877 DEBUG [StoreCloser-TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/B/2f83a6476b6942f5aa1def86cf7ec5f3 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/B/2f83a6476b6942f5aa1def86cf7ec5f3 2024-11-27T12:27:22,878 DEBUG [StoreCloser-TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/B/88bdec78876e48c79190c8c2d6ceb92e to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/B/88bdec78876e48c79190c8c2d6ceb92e 2024-11-27T12:27:22,879 DEBUG [StoreCloser-TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/B/285d3e9acf9d40fe85482aebd90592bc to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/B/285d3e9acf9d40fe85482aebd90592bc 2024-11-27T12:27:22,880 DEBUG [StoreCloser-TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/B/f385111ce4224651948b03094395a174 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/B/f385111ce4224651948b03094395a174 2024-11-27T12:27:22,880 DEBUG [StoreCloser-TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/B/6dba99719e6040cea7926a438bd87862 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/B/6dba99719e6040cea7926a438bd87862 2024-11-27T12:27:22,881 DEBUG [StoreCloser-TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/B/d5888bdca7e44e228c8a4de310e008fb to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/B/d5888bdca7e44e228c8a4de310e008fb 2024-11-27T12:27:22,882 DEBUG [StoreCloser-TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/C/16dc89550181433ca2d05b2cfe013ebc, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/C/7f4cf58d304f4f888e3306fc9a2648f1, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/C/979cc434a6d64166a2b6fb9296f915b5, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/C/2a2b141351ca4b2a85a2509b2be79e24, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/C/05ab749ddd5940548076ec08fbf75af8, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/C/b2c41d0741fd48ce85f1ddf020eed9fd, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/C/7fa7e87809084578bb0b283ab73c8d84, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/C/a1d88299e8e94c7d94c6349417cc0e14, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/C/91c2ab16cb8e4cd79010a5626c7fc4e3, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/C/e7c83f80aa514409a605b299d757738f, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/C/66e7f70f029b4c9c9a7137cd07a83733, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/C/22ffae6bc9834fd1b1abf41f55c812d2, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/C/8d3207247622435aac8158df1d185eae, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/C/962bf29d5ff04658bab0af426812e61b, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/C/f1e4a25c3d4949c991002be4c832d312, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/C/8bab4586020d4d5984d6bb534d889caa, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/C/c28f2d6b092543e9a056079cbf6e8688, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/C/1f638e7f924c436f8bfc5a0a54b41eae, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/C/e48133d50c3944ad88005edde347376e, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/C/48df09131d674cb8b4d94e260d35c9ea] to archive 2024-11-27T12:27:22,883 DEBUG [StoreCloser-TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-27T12:27:22,884 DEBUG [StoreCloser-TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/C/16dc89550181433ca2d05b2cfe013ebc to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/C/16dc89550181433ca2d05b2cfe013ebc 2024-11-27T12:27:22,885 DEBUG [StoreCloser-TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/C/7f4cf58d304f4f888e3306fc9a2648f1 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/C/7f4cf58d304f4f888e3306fc9a2648f1 2024-11-27T12:27:22,886 DEBUG [StoreCloser-TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/C/979cc434a6d64166a2b6fb9296f915b5 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/C/979cc434a6d64166a2b6fb9296f915b5 2024-11-27T12:27:22,886 DEBUG [StoreCloser-TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/C/2a2b141351ca4b2a85a2509b2be79e24 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/C/2a2b141351ca4b2a85a2509b2be79e24 2024-11-27T12:27:22,887 DEBUG [StoreCloser-TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/C/05ab749ddd5940548076ec08fbf75af8 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/C/05ab749ddd5940548076ec08fbf75af8 2024-11-27T12:27:22,888 DEBUG [StoreCloser-TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/C/b2c41d0741fd48ce85f1ddf020eed9fd to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/C/b2c41d0741fd48ce85f1ddf020eed9fd 2024-11-27T12:27:22,889 DEBUG [StoreCloser-TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/C/7fa7e87809084578bb0b283ab73c8d84 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/C/7fa7e87809084578bb0b283ab73c8d84 2024-11-27T12:27:22,890 DEBUG [StoreCloser-TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/C/a1d88299e8e94c7d94c6349417cc0e14 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/C/a1d88299e8e94c7d94c6349417cc0e14 2024-11-27T12:27:22,890 DEBUG [StoreCloser-TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/C/91c2ab16cb8e4cd79010a5626c7fc4e3 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/C/91c2ab16cb8e4cd79010a5626c7fc4e3 2024-11-27T12:27:22,891 DEBUG [StoreCloser-TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/C/e7c83f80aa514409a605b299d757738f to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/C/e7c83f80aa514409a605b299d757738f 2024-11-27T12:27:22,892 DEBUG [StoreCloser-TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/C/66e7f70f029b4c9c9a7137cd07a83733 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/C/66e7f70f029b4c9c9a7137cd07a83733 2024-11-27T12:27:22,892 DEBUG [StoreCloser-TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/C/22ffae6bc9834fd1b1abf41f55c812d2 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/C/22ffae6bc9834fd1b1abf41f55c812d2 2024-11-27T12:27:22,893 DEBUG [StoreCloser-TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/C/8d3207247622435aac8158df1d185eae to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/C/8d3207247622435aac8158df1d185eae 2024-11-27T12:27:22,893 DEBUG [StoreCloser-TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/C/962bf29d5ff04658bab0af426812e61b to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/C/962bf29d5ff04658bab0af426812e61b 2024-11-27T12:27:22,894 DEBUG [StoreCloser-TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/C/f1e4a25c3d4949c991002be4c832d312 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/C/f1e4a25c3d4949c991002be4c832d312 2024-11-27T12:27:22,894 DEBUG [StoreCloser-TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/C/8bab4586020d4d5984d6bb534d889caa to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/C/8bab4586020d4d5984d6bb534d889caa 2024-11-27T12:27:22,895 DEBUG [StoreCloser-TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/C/c28f2d6b092543e9a056079cbf6e8688 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/C/c28f2d6b092543e9a056079cbf6e8688 2024-11-27T12:27:22,896 DEBUG [StoreCloser-TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/C/1f638e7f924c436f8bfc5a0a54b41eae to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/C/1f638e7f924c436f8bfc5a0a54b41eae 2024-11-27T12:27:22,896 DEBUG [StoreCloser-TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/C/e48133d50c3944ad88005edde347376e to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/C/e48133d50c3944ad88005edde347376e 2024-11-27T12:27:22,897 DEBUG [StoreCloser-TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/C/48df09131d674cb8b4d94e260d35c9ea to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/C/48df09131d674cb8b4d94e260d35c9ea 2024-11-27T12:27:22,900 DEBUG [RS_CLOSE_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/recovered.edits/314.seqid, newMaxSeqId=314, maxSeqId=4 2024-11-27T12:27:22,900 INFO [RS_CLOSE_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095. 2024-11-27T12:27:22,900 DEBUG [RS_CLOSE_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HRegion(1676): Region close journal for a2c58e0d0c97538318beef21ebb34095: Waiting for close lock at 1732710441153Running coprocessor pre-close hooks at 1732710441153Disabling compacts and flushes for region at 1732710441153Disabling writes for close at 1732710441153Obtaining lock to block concurrent updates at 1732710441154 (+1 ms)Preparing flush snapshotting stores in a2c58e0d0c97538318beef21ebb34095 at 1732710441154Finished memstore snapshotting TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095., syncing WAL and waiting on mvcc, flushsize=dataSize=13740, getHeapSize=36720, getOffHeapSize=0, getCellsCount=300 at 1732710441154Flushing stores of TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095. at 1732710441155 (+1 ms)Flushing a2c58e0d0c97538318beef21ebb34095/A: creating writer at 1732710441155Flushing mob file a2c58e0d0c97538318beef21ebb34095/A: appending metadata at 1732710441165 (+10 ms)Flushing mob file a2c58e0d0c97538318beef21ebb34095/A: closing flushed file at 1732710441165Flushing a2c58e0d0c97538318beef21ebb34095/A: appending metadata at 1732710441580 (+415 ms)Flushing a2c58e0d0c97538318beef21ebb34095/A: closing flushed file at 1732710441580Flushing a2c58e0d0c97538318beef21ebb34095/B: creating writer at 1732710441997 (+417 ms)Flushing a2c58e0d0c97538318beef21ebb34095/B: appending metadata at 1732710442000 (+3 ms)Flushing a2c58e0d0c97538318beef21ebb34095/B: closing flushed file at 1732710442000Flushing a2c58e0d0c97538318beef21ebb34095/C: creating writer at 1732710442414 (+414 ms)Flushing a2c58e0d0c97538318beef21ebb34095/C: appending metadata at 1732710442417 (+3 ms)Flushing a2c58e0d0c97538318beef21ebb34095/C: closing flushed file at 1732710442417Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@14fbecd9: reopening flushed file at 1732710442831 (+414 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@594043b6: reopening flushed file at 1732710442838 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@50a2320: reopening flushed file at 1732710442843 (+5 ms)Finished flush of dataSize ~13.42 KB/13740, heapSize ~35.86 KB/36720, currentSize=0 B/0 for a2c58e0d0c97538318beef21ebb34095 in 1694ms, sequenceid=311, compaction requested=false at 1732710442847 (+4 ms)Writing region close event to WAL at 1732710442898 (+51 ms)Running coprocessor post-close hooks at 1732710442900 (+2 ms)Closed at 1732710442900 2024-11-27T12:27:22,902 INFO [RS_CLOSE_REGION-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] handler.UnassignRegionHandler(157): Closed a2c58e0d0c97538318beef21ebb34095 2024-11-27T12:27:22,902 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=170 updating hbase:meta row=a2c58e0d0c97538318beef21ebb34095, regionState=CLOSED 2024-11-27T12:27:22,903 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=171, ppid=170, state=RUNNABLE, hasLock=false; CloseRegionProcedure a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 because future has completed 2024-11-27T12:27:22,905 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=171, resume processing ppid=170 2024-11-27T12:27:22,905 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=171, ppid=170, state=SUCCESS, hasLock=false; CloseRegionProcedure a2c58e0d0c97538318beef21ebb34095, server=1ba5af43fd99,36251,1732710241631 in 1.9050 sec 2024-11-27T12:27:22,906 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=170, resume processing ppid=169 2024-11-27T12:27:22,906 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=170, ppid=169, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a2c58e0d0c97538318beef21ebb34095, UNASSIGN in 1.9090 sec 2024-11-27T12:27:22,907 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=169, resume processing ppid=168 2024-11-27T12:27:22,907 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=169, ppid=168, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.9120 sec 2024-11-27T12:27:22,908 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732710442908"}]},"ts":"1732710442908"} 2024-11-27T12:27:22,909 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-27T12:27:22,909 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-27T12:27:22,910 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=168, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.9230 sec 2024-11-27T12:27:23,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-27T12:27:23,149 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-27T12:27:23,151 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36215 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-11-27T12:27:23,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36215 {}] procedure2.ProcedureExecutor(1139): Stored pid=172, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-27T12:27:23,155 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=172, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-27T12:27:23,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=172 2024-11-27T12:27:23,156 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=172, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-27T12:27:23,161 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095 2024-11-27T12:27:23,164 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/A, FileablePath, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/B, FileablePath, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/C, FileablePath, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/recovered.edits] 2024-11-27T12:27:23,167 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/A/83d3111070984a6d9fc782086ffbbb57 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/A/83d3111070984a6d9fc782086ffbbb57 2024-11-27T12:27:23,168 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/A/8af8aca92f844976aae7072a306dae45 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/A/8af8aca92f844976aae7072a306dae45 2024-11-27T12:27:23,171 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/B/22692424d5404ef4a157e34df92d5a39 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/B/22692424d5404ef4a157e34df92d5a39 2024-11-27T12:27:23,172 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/B/8e1685d17568489593bc12c7c49d9b96 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/B/8e1685d17568489593bc12c7c49d9b96 2024-11-27T12:27:23,176 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/C/532ddce2f8844385b7d1e03f94e0045f to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/C/532ddce2f8844385b7d1e03f94e0045f 2024-11-27T12:27:23,177 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/C/ce930e4d820e44c7a5a03dea420f9f0b to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/C/ce930e4d820e44c7a5a03dea420f9f0b 2024-11-27T12:27:23,181 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/recovered.edits/314.seqid to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095/recovered.edits/314.seqid 2024-11-27T12:27:23,182 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/default/TestAcidGuarantees/a2c58e0d0c97538318beef21ebb34095 2024-11-27T12:27:23,182 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-27T12:27:23,183 DEBUG [PEWorker-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-27T12:27:23,183 DEBUG [PEWorker-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-27T12:27:23,185 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127129f6806ca0d4f3d838409ec1c2c84aa_a2c58e0d0c97538318beef21ebb34095 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127129f6806ca0d4f3d838409ec1c2c84aa_a2c58e0d0c97538318beef21ebb34095 2024-11-27T12:27:23,186 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411271605ba86542c459aaa6089464035a839_a2c58e0d0c97538318beef21ebb34095 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411271605ba86542c459aaa6089464035a839_a2c58e0d0c97538318beef21ebb34095 2024-11-27T12:27:23,187 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411272893a15ee2a8493da2b6dd136649e7a4_a2c58e0d0c97538318beef21ebb34095 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411272893a15ee2a8493da2b6dd136649e7a4_a2c58e0d0c97538318beef21ebb34095 2024-11-27T12:27:23,187 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112738f71963cbfe469ab69da02c97c2e8e7_a2c58e0d0c97538318beef21ebb34095 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112738f71963cbfe469ab69da02c97c2e8e7_a2c58e0d0c97538318beef21ebb34095 2024-11-27T12:27:23,188 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112744ef427e875a41bfaea84833ae563daa_a2c58e0d0c97538318beef21ebb34095 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112744ef427e875a41bfaea84833ae563daa_a2c58e0d0c97538318beef21ebb34095 2024-11-27T12:27:23,189 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127668429895811467f8c9bb86d93a6eb11_a2c58e0d0c97538318beef21ebb34095 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127668429895811467f8c9bb86d93a6eb11_a2c58e0d0c97538318beef21ebb34095 2024-11-27T12:27:23,189 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127823ad32546f04bbc9b8bf3e9eafd1a19_a2c58e0d0c97538318beef21ebb34095 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127823ad32546f04bbc9b8bf3e9eafd1a19_a2c58e0d0c97538318beef21ebb34095 2024-11-27T12:27:23,190 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112795ab0450871342d0b7077743b306df6a_a2c58e0d0c97538318beef21ebb34095 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112795ab0450871342d0b7077743b306df6a_a2c58e0d0c97538318beef21ebb34095 2024-11-27T12:27:23,190 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127972984d15b714cf58a203ea1b5f1ecfe_a2c58e0d0c97538318beef21ebb34095 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127972984d15b714cf58a203ea1b5f1ecfe_a2c58e0d0c97538318beef21ebb34095 2024-11-27T12:27:23,191 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127ac250f6c02cc4f50bdb53177d111f030_a2c58e0d0c97538318beef21ebb34095 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127ac250f6c02cc4f50bdb53177d111f030_a2c58e0d0c97538318beef21ebb34095 2024-11-27T12:27:23,192 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127b5e9cf79e3614698808f7b7f9dbf3d7b_a2c58e0d0c97538318beef21ebb34095 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127b5e9cf79e3614698808f7b7f9dbf3d7b_a2c58e0d0c97538318beef21ebb34095 2024-11-27T12:27:23,193 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127ccade85c6fd24b3abba1726bd51ef502_a2c58e0d0c97538318beef21ebb34095 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127ccade85c6fd24b3abba1726bd51ef502_a2c58e0d0c97538318beef21ebb34095 2024-11-27T12:27:23,193 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127d0bafcb00d7c476aac2827178eaddcf1_a2c58e0d0c97538318beef21ebb34095 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127d0bafcb00d7c476aac2827178eaddcf1_a2c58e0d0c97538318beef21ebb34095 2024-11-27T12:27:23,194 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127e3d48b15db7d4e008af80505ae19ebe6_a2c58e0d0c97538318beef21ebb34095 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127e3d48b15db7d4e008af80505ae19ebe6_a2c58e0d0c97538318beef21ebb34095 2024-11-27T12:27:23,195 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127e5a1f0fec8824a73b09559be94e0d16e_a2c58e0d0c97538318beef21ebb34095 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127e5a1f0fec8824a73b09559be94e0d16e_a2c58e0d0c97538318beef21ebb34095 2024-11-27T12:27:23,196 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127e915390bfe974e05a2b056f275272e00_a2c58e0d0c97538318beef21ebb34095 to hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127e915390bfe974e05a2b056f275272e00_a2c58e0d0c97538318beef21ebb34095 2024-11-27T12:27:23,197 DEBUG [PEWorker-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-27T12:27:23,199 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=172, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-27T12:27:23,201 WARN [PEWorker-3 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-27T12:27:23,203 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-27T12:27:23,204 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=172, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-27T12:27:23,204 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-27T12:27:23,204 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732710443204"}]},"ts":"9223372036854775807"} 2024-11-27T12:27:23,206 INFO [PEWorker-3 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-27T12:27:23,206 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => a2c58e0d0c97538318beef21ebb34095, NAME => 'TestAcidGuarantees,,1732710409434.a2c58e0d0c97538318beef21ebb34095.', STARTKEY => '', ENDKEY => ''}] 2024-11-27T12:27:23,206 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-27T12:27:23,206 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732710443206"}]},"ts":"9223372036854775807"} 2024-11-27T12:27:23,208 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-27T12:27:23,208 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(133): Finished pid=172, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-27T12:27:23,210 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=172, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 57 msec 2024-11-27T12:27:23,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=172 2024-11-27T12:27:23,269 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-27T12:27:23,270 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-27T12:27:23,284 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMobGetAtomicity Thread=237 (was 237), OpenFileDescriptor=451 (was 441) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=334 (was 372), ProcessCount=11 (was 11), AvailableMemoryMB=8634 (was 8650) 2024-11-27T12:27:23,284 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-27T12:27:23,284 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-27T12:27:23,284 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-27T12:27:23,284 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T12:27:23,284 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T12:27:23,284 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T12:27:23,285 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-27T12:27:23,285 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1201124139, stopped=false 2024-11-27T12:27:23,285 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=1ba5af43fd99,36215,1732710240860 2024-11-27T12:27:23,347 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36215-0x1017c400ae90000, quorum=127.0.0.1:62785, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-27T12:27:23,347 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36251-0x1017c400ae90001, quorum=127.0.0.1:62785, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-27T12:27:23,347 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36251-0x1017c400ae90001, quorum=127.0.0.1:62785, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T12:27:23,347 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36215-0x1017c400ae90000, quorum=127.0.0.1:62785, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T12:27:23,347 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-27T12:27:23,348 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-27T12:27:23,349 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-27T12:27:23,349 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:36215-0x1017c400ae90000, quorum=127.0.0.1:62785, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-27T12:27:23,349 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T12:27:23,350 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:36251-0x1017c400ae90001, quorum=127.0.0.1:62785, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-27T12:27:23,350 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '1ba5af43fd99,36251,1732710241631' ***** 2024-11-27T12:27:23,350 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-27T12:27:23,351 INFO [RS:0;1ba5af43fd99:36251 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-27T12:27:23,351 INFO [RS:0;1ba5af43fd99:36251 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-27T12:27:23,351 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-27T12:27:23,352 INFO [RS:0;1ba5af43fd99:36251 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-27T12:27:23,352 INFO [RS:0;1ba5af43fd99:36251 {}] regionserver.HRegionServer(959): stopping server 1ba5af43fd99,36251,1732710241631 2024-11-27T12:27:23,352 INFO [RS:0;1ba5af43fd99:36251 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-27T12:27:23,352 INFO [RS:0;1ba5af43fd99:36251 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;1ba5af43fd99:36251. 2024-11-27T12:27:23,352 DEBUG [RS:0;1ba5af43fd99:36251 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-27T12:27:23,352 DEBUG [RS:0;1ba5af43fd99:36251 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T12:27:23,352 INFO [RS:0;1ba5af43fd99:36251 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-27T12:27:23,352 INFO [RS:0;1ba5af43fd99:36251 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-27T12:27:23,352 INFO [RS:0;1ba5af43fd99:36251 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-27T12:27:23,352 INFO [RS:0;1ba5af43fd99:36251 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-27T12:27:23,353 INFO [RS:0;1ba5af43fd99:36251 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-27T12:27:23,353 DEBUG [RS:0;1ba5af43fd99:36251 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-11-27T12:27:23,354 DEBUG [RS:0;1ba5af43fd99:36251 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-27T12:27:23,354 DEBUG [RS_CLOSE_META-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-27T12:27:23,354 INFO [RS_CLOSE_META-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-27T12:27:23,354 DEBUG [RS_CLOSE_META-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-27T12:27:23,354 DEBUG [RS_CLOSE_META-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-27T12:27:23,354 DEBUG [RS_CLOSE_META-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-27T12:27:23,354 INFO [RS_CLOSE_META-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=19.91 KB heapSize=35.20 KB 2024-11-27T12:27:23,374 DEBUG [RS_CLOSE_META-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/hbase/meta/1588230740/.tmp/info/9f629555be0c40d1b093f0fdc79c5ea9 is 95, key is TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4./info:/1732710280031/DeleteFamily/seqid=0 2024-11-27T12:27:23,377 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742432_1608 (size=6514) 2024-11-27T12:27:23,419 INFO [regionserver/1ba5af43fd99:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-27T12:27:23,554 DEBUG [RS:0;1ba5af43fd99:36251 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-27T12:27:23,720 INFO [regionserver/1ba5af43fd99:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-27T12:27:23,720 INFO [regionserver/1ba5af43fd99:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-27T12:27:23,754 DEBUG [RS:0;1ba5af43fd99:36251 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-27T12:27:23,778 INFO [RS_CLOSE_META-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=16.75 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/hbase/meta/1588230740/.tmp/info/9f629555be0c40d1b093f0fdc79c5ea9 2024-11-27T12:27:23,802 DEBUG [RS_CLOSE_META-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/hbase/meta/1588230740/.tmp/ns/f9d84b2112ec4642ae77816e601afa28 is 93, key is TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4./ns:/1732710279988/DeleteFamily/seqid=0 2024-11-27T12:27:23,805 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742433_1609 (size=5906) 2024-11-27T12:27:23,954 DEBUG [RS:0;1ba5af43fd99:36251 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-27T12:27:24,155 DEBUG [RS:0;1ba5af43fd99:36251 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-27T12:27:24,206 INFO [RS_CLOSE_META-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=608 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/hbase/meta/1588230740/.tmp/ns/f9d84b2112ec4642ae77816e601afa28 2024-11-27T12:27:24,228 DEBUG [RS_CLOSE_META-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/hbase/meta/1588230740/.tmp/rep_barrier/900bfb2835e64f218b41788933343827 is 102, key is TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4./rep_barrier:/1732710279988/DeleteFamily/seqid=0 2024-11-27T12:27:24,231 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742434_1610 (size=6025) 2024-11-27T12:27:24,355 DEBUG [RS:0;1ba5af43fd99:36251 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-27T12:27:24,555 DEBUG [RS:0;1ba5af43fd99:36251 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-27T12:27:24,633 INFO [RS_CLOSE_META-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=588 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/hbase/meta/1588230740/.tmp/rep_barrier/900bfb2835e64f218b41788933343827 2024-11-27T12:27:24,660 DEBUG [RS_CLOSE_META-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/hbase/meta/1588230740/.tmp/table/cf2eb6cf8ec749adbe7726086052162c is 96, key is TestAcidGuarantees,,1732710246285.953f259f29896493f17802c953dd7fd4./table:/1732710279988/DeleteFamily/seqid=0 2024-11-27T12:27:24,663 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742435_1611 (size=5936) 2024-11-27T12:27:24,755 DEBUG [RS:0;1ba5af43fd99:36251 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-27T12:27:24,956 DEBUG [RS:0;1ba5af43fd99:36251 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-27T12:27:25,064 INFO [RS_CLOSE_META-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.99 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/hbase/meta/1588230740/.tmp/table/cf2eb6cf8ec749adbe7726086052162c 2024-11-27T12:27:25,074 DEBUG [RS_CLOSE_META-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/hbase/meta/1588230740/.tmp/info/9f629555be0c40d1b093f0fdc79c5ea9 as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/hbase/meta/1588230740/info/9f629555be0c40d1b093f0fdc79c5ea9 2024-11-27T12:27:25,079 INFO [RS_CLOSE_META-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/hbase/meta/1588230740/info/9f629555be0c40d1b093f0fdc79c5ea9, entries=12, sequenceid=90, filesize=6.4 K 2024-11-27T12:27:25,080 DEBUG [RS_CLOSE_META-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/hbase/meta/1588230740/.tmp/ns/f9d84b2112ec4642ae77816e601afa28 as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/hbase/meta/1588230740/ns/f9d84b2112ec4642ae77816e601afa28 2024-11-27T12:27:25,085 INFO [RS_CLOSE_META-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/hbase/meta/1588230740/ns/f9d84b2112ec4642ae77816e601afa28, entries=8, sequenceid=90, filesize=5.8 K 2024-11-27T12:27:25,086 DEBUG [RS_CLOSE_META-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/hbase/meta/1588230740/.tmp/rep_barrier/900bfb2835e64f218b41788933343827 as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/hbase/meta/1588230740/rep_barrier/900bfb2835e64f218b41788933343827 2024-11-27T12:27:25,089 INFO [RS_CLOSE_META-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/hbase/meta/1588230740/rep_barrier/900bfb2835e64f218b41788933343827, entries=6, sequenceid=90, filesize=5.9 K 2024-11-27T12:27:25,090 DEBUG [RS_CLOSE_META-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/hbase/meta/1588230740/.tmp/table/cf2eb6cf8ec749adbe7726086052162c as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/hbase/meta/1588230740/table/cf2eb6cf8ec749adbe7726086052162c 2024-11-27T12:27:25,093 INFO [RS_CLOSE_META-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/hbase/meta/1588230740/table/cf2eb6cf8ec749adbe7726086052162c, entries=7, sequenceid=90, filesize=5.8 K 2024-11-27T12:27:25,094 INFO [RS_CLOSE_META-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1740ms, sequenceid=90, compaction requested=false 2024-11-27T12:27:25,098 DEBUG [RS_CLOSE_META-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/data/hbase/meta/1588230740/recovered.edits/93.seqid, newMaxSeqId=93, maxSeqId=1 2024-11-27T12:27:25,099 DEBUG [RS_CLOSE_META-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-27T12:27:25,099 INFO [RS_CLOSE_META-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-27T12:27:25,099 DEBUG [RS_CLOSE_META-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732710443353Running coprocessor pre-close hooks at 1732710443354 (+1 ms)Disabling compacts and flushes for region at 1732710443354Disabling writes for close at 1732710443354Obtaining lock to block concurrent updates at 1732710443354Preparing flush snapshotting stores in 1588230740 at 1732710443354Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=20384, getHeapSize=35984, getOffHeapSize=0, getCellsCount=191 at 1732710443354Flushing stores of hbase:meta,,1.1588230740 at 1732710443355 (+1 ms)Flushing 1588230740/info: creating writer at 1732710443355Flushing 1588230740/info: appending metadata at 1732710443373 (+18 ms)Flushing 1588230740/info: closing flushed file at 1732710443373Flushing 1588230740/ns: creating writer at 1732710443786 (+413 ms)Flushing 1588230740/ns: appending metadata at 1732710443802 (+16 ms)Flushing 1588230740/ns: closing flushed file at 1732710443802Flushing 1588230740/rep_barrier: creating writer at 1732710444210 (+408 ms)Flushing 1588230740/rep_barrier: appending metadata at 1732710444228 (+18 ms)Flushing 1588230740/rep_barrier: closing flushed file at 1732710444228Flushing 1588230740/table: creating writer at 1732710444642 (+414 ms)Flushing 1588230740/table: appending metadata at 1732710444660 (+18 ms)Flushing 1588230740/table: closing flushed file at 1732710444660Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@21a51fd7: reopening flushed file at 1732710445073 (+413 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3b04d62e: reopening flushed file at 1732710445079 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@66fcf21f: reopening flushed file at 1732710445085 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@666975e3: reopening flushed file at 1732710445089 (+4 ms)Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1740ms, sequenceid=90, compaction requested=false at 1732710445094 (+5 ms)Writing region close event to WAL at 1732710445095 (+1 ms)Running coprocessor post-close hooks at 1732710445099 (+4 ms)Closed at 1732710445099 2024-11-27T12:27:25,099 DEBUG [RS_CLOSE_META-regionserver/1ba5af43fd99:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-27T12:27:25,156 INFO [RS:0;1ba5af43fd99:36251 {}] regionserver.HRegionServer(976): stopping server 1ba5af43fd99,36251,1732710241631; all regions closed. 2024-11-27T12:27:25,164 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073741834_1010 (size=25344) 2024-11-27T12:27:25,169 DEBUG [RS:0;1ba5af43fd99:36251 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/oldWALs 2024-11-27T12:27:25,169 INFO [RS:0;1ba5af43fd99:36251 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 1ba5af43fd99%2C36251%2C1732710241631.meta:.meta(num 1732710245812) 2024-11-27T12:27:25,172 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073741832_1008 (size=13922868) 2024-11-27T12:27:25,175 DEBUG [RS:0;1ba5af43fd99:36251 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/oldWALs 2024-11-27T12:27:25,175 INFO [RS:0;1ba5af43fd99:36251 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 1ba5af43fd99%2C36251%2C1732710241631:(num 1732710244827) 2024-11-27T12:27:25,175 DEBUG [RS:0;1ba5af43fd99:36251 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T12:27:25,175 INFO [RS:0;1ba5af43fd99:36251 {}] regionserver.LeaseManager(133): Closed leases 2024-11-27T12:27:25,175 INFO [RS:0;1ba5af43fd99:36251 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-27T12:27:25,176 INFO [RS:0;1ba5af43fd99:36251 {}] hbase.ChoreService(370): Chore service for: regionserver/1ba5af43fd99:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-11-27T12:27:25,176 INFO [RS:0;1ba5af43fd99:36251 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-27T12:27:25,176 INFO [regionserver/1ba5af43fd99:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-27T12:27:25,177 INFO [RS:0;1ba5af43fd99:36251 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:36251 2024-11-27T12:27:25,241 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36251-0x1017c400ae90001, quorum=127.0.0.1:62785, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/1ba5af43fd99,36251,1732710241631 2024-11-27T12:27:25,241 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36215-0x1017c400ae90000, quorum=127.0.0.1:62785, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-27T12:27:25,241 INFO [RS:0;1ba5af43fd99:36251 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-27T12:27:25,292 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [1ba5af43fd99,36251,1732710241631] 2024-11-27T12:27:25,314 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/1ba5af43fd99,36251,1732710241631 already deleted, retry=false 2024-11-27T12:27:25,314 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 1ba5af43fd99,36251,1732710241631 expired; onlineServers=0 2024-11-27T12:27:25,315 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '1ba5af43fd99,36215,1732710240860' ***** 2024-11-27T12:27:25,315 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-27T12:27:25,315 INFO [M:0;1ba5af43fd99:36215 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-27T12:27:25,315 INFO [M:0;1ba5af43fd99:36215 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-27T12:27:25,315 DEBUG [M:0;1ba5af43fd99:36215 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-27T12:27:25,315 DEBUG [M:0;1ba5af43fd99:36215 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-27T12:27:25,315 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-27T12:27:25,315 DEBUG [master/1ba5af43fd99:0:becomeActiveMaster-HFileCleaner.large.0-1732710244521 {}] cleaner.HFileCleaner(306): Exit Thread[master/1ba5af43fd99:0:becomeActiveMaster-HFileCleaner.large.0-1732710244521,5,FailOnTimeoutGroup] 2024-11-27T12:27:25,315 DEBUG [master/1ba5af43fd99:0:becomeActiveMaster-HFileCleaner.small.0-1732710244522 {}] cleaner.HFileCleaner(306): Exit Thread[master/1ba5af43fd99:0:becomeActiveMaster-HFileCleaner.small.0-1732710244522,5,FailOnTimeoutGroup] 2024-11-27T12:27:25,316 INFO [M:0;1ba5af43fd99:36215 {}] hbase.ChoreService(370): Chore service for: master/1ba5af43fd99:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-27T12:27:25,316 INFO [M:0;1ba5af43fd99:36215 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-27T12:27:25,316 DEBUG [M:0;1ba5af43fd99:36215 {}] master.HMaster(1795): Stopping service threads 2024-11-27T12:27:25,316 INFO [M:0;1ba5af43fd99:36215 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-27T12:27:25,316 INFO [M:0;1ba5af43fd99:36215 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-27T12:27:25,316 ERROR [M:0;1ba5af43fd99:36215 {}] procedure2.ProcedureExecutor(763): There are still active thread in group java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10], see STDOUT java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] Thread[IPC Client (935358161) connection to localhost/127.0.0.1:45125 from jenkins,5,PEWorkerGroup] Thread[IPC Parameter Sending Thread for localhost/127.0.0.1:45125,5,PEWorkerGroup] Thread[HFileArchiver-6,5,PEWorkerGroup] 2024-11-27T12:27:25,318 INFO [M:0;1ba5af43fd99:36215 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-27T12:27:25,318 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-27T12:27:25,325 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36215-0x1017c400ae90000, quorum=127.0.0.1:62785, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-27T12:27:25,325 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36215-0x1017c400ae90000, quorum=127.0.0.1:62785, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T12:27:25,325 DEBUG [M:0;1ba5af43fd99:36215 {}] zookeeper.ZKUtil(347): master:36215-0x1017c400ae90000, quorum=127.0.0.1:62785, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-27T12:27:25,326 WARN [M:0;1ba5af43fd99:36215 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-27T12:27:25,327 INFO [M:0;1ba5af43fd99:36215 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/.lastflushedseqids 2024-11-27T12:27:25,340 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742436_1612 (size=72) 2024-11-27T12:27:25,393 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36251-0x1017c400ae90001, quorum=127.0.0.1:62785, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-27T12:27:25,393 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36251-0x1017c400ae90001, quorum=127.0.0.1:62785, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-27T12:27:25,393 INFO [RS:0;1ba5af43fd99:36251 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-27T12:27:25,394 INFO [RS:0;1ba5af43fd99:36251 {}] regionserver.HRegionServer(1031): Exiting; stopping=1ba5af43fd99,36251,1732710241631; zookeeper connection closed. 2024-11-27T12:27:25,394 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@68231a95 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@68231a95 2024-11-27T12:27:25,395 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-11-27T12:27:25,742 INFO [M:0;1ba5af43fd99:36215 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-27T12:27:25,742 INFO [M:0;1ba5af43fd99:36215 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-27T12:27:25,742 DEBUG [M:0;1ba5af43fd99:36215 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-27T12:27:25,743 INFO [M:0;1ba5af43fd99:36215 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-27T12:27:25,743 DEBUG [M:0;1ba5af43fd99:36215 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-27T12:27:25,743 DEBUG [M:0;1ba5af43fd99:36215 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-27T12:27:25,743 DEBUG [M:0;1ba5af43fd99:36215 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-27T12:27:25,743 INFO [M:0;1ba5af43fd99:36215 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=742.48 KB heapSize=911.38 KB 2024-11-27T12:27:25,765 DEBUG [M:0;1ba5af43fd99:36215 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/f4302fe2863e4386ad541b31466c3a31 is 82, key is hbase:meta,,1/info:regioninfo/1732710245898/Put/seqid=0 2024-11-27T12:27:25,768 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742437_1613 (size=5672) 2024-11-27T12:27:26,170 INFO [M:0;1ba5af43fd99:36215 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=2077 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/f4302fe2863e4386ad541b31466c3a31 2024-11-27T12:27:26,201 DEBUG [M:0;1ba5af43fd99:36215 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/a5339507b3d243c4a1d68ba37d0cede6 is 2276, key is \x00\x00\x00\x00\x00\x00\x00\x97/proc:d/1732710412545/Put/seqid=0 2024-11-27T12:27:26,205 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742438_1614 (size=44542) 2024-11-27T12:27:26,606 INFO [M:0;1ba5af43fd99:36215 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=741.93 KB at sequenceid=2077 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/a5339507b3d243c4a1d68ba37d0cede6 2024-11-27T12:27:26,615 INFO [M:0;1ba5af43fd99:36215 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for a5339507b3d243c4a1d68ba37d0cede6 2024-11-27T12:27:26,634 DEBUG [M:0;1ba5af43fd99:36215 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/21ac7ee68d8b4ef0aaac41d736bc59ee is 69, key is 1ba5af43fd99,36251,1732710241631/rs:state/1732710244582/Put/seqid=0 2024-11-27T12:27:26,637 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073742439_1615 (size=5156) 2024-11-27T12:27:27,039 INFO [M:0;1ba5af43fd99:36215 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=2077 (bloomFilter=true), to=hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/21ac7ee68d8b4ef0aaac41d736bc59ee 2024-11-27T12:27:27,051 DEBUG [M:0;1ba5af43fd99:36215 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/f4302fe2863e4386ad541b31466c3a31 as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/f4302fe2863e4386ad541b31466c3a31 2024-11-27T12:27:27,054 INFO [M:0;1ba5af43fd99:36215 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/f4302fe2863e4386ad541b31466c3a31, entries=8, sequenceid=2077, filesize=5.5 K 2024-11-27T12:27:27,054 DEBUG [M:0;1ba5af43fd99:36215 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/a5339507b3d243c4a1d68ba37d0cede6 as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/a5339507b3d243c4a1d68ba37d0cede6 2024-11-27T12:27:27,057 INFO [M:0;1ba5af43fd99:36215 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for a5339507b3d243c4a1d68ba37d0cede6 2024-11-27T12:27:27,057 INFO [M:0;1ba5af43fd99:36215 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/a5339507b3d243c4a1d68ba37d0cede6, entries=172, sequenceid=2077, filesize=43.5 K 2024-11-27T12:27:27,057 DEBUG [M:0;1ba5af43fd99:36215 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/21ac7ee68d8b4ef0aaac41d736bc59ee as hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/21ac7ee68d8b4ef0aaac41d736bc59ee 2024-11-27T12:27:27,060 INFO [M:0;1ba5af43fd99:36215 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45125/user/jenkins/test-data/56ad9740-399d-c8e8-9480-1d502bf6e02c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/21ac7ee68d8b4ef0aaac41d736bc59ee, entries=1, sequenceid=2077, filesize=5.0 K 2024-11-27T12:27:27,061 INFO [M:0;1ba5af43fd99:36215 {}] regionserver.HRegion(3140): Finished flush of dataSize ~742.48 KB/760301, heapSize ~911.09 KB/932952, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1317ms, sequenceid=2077, compaction requested=false 2024-11-27T12:27:27,062 INFO [M:0;1ba5af43fd99:36215 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-27T12:27:27,062 DEBUG [M:0;1ba5af43fd99:36215 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732710445742Disabling compacts and flushes for region at 1732710445742Disabling writes for close at 1732710445743 (+1 ms)Obtaining lock to block concurrent updates at 1732710445743Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1732710445743Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=760301, getHeapSize=933192, getOffHeapSize=0, getCellsCount=2274 at 1732710445744 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1732710445745 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1732710445746 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1732710445765 (+19 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1732710445765Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1732710446179 (+414 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1732710446201 (+22 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1732710446201Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1732710446615 (+414 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1732710446634 (+19 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1732710446634Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@25ff1293: reopening flushed file at 1732710447048 (+414 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3084b389: reopening flushed file at 1732710447054 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@34bff4cc: reopening flushed file at 1732710447057 (+3 ms)Finished flush of dataSize ~742.48 KB/760301, heapSize ~911.09 KB/932952, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1317ms, sequenceid=2077, compaction requested=false at 1732710447061 (+4 ms)Writing region close event to WAL at 1732710447062 (+1 ms)Closed at 1732710447062 2024-11-27T12:27:27,064 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37513 is added to blk_1073741830_1006 (size=896513) 2024-11-27T12:27:27,065 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-27T12:27:27,065 INFO [M:0;1ba5af43fd99:36215 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-27T12:27:27,065 INFO [M:0;1ba5af43fd99:36215 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:36215 2024-11-27T12:27:27,065 INFO [M:0;1ba5af43fd99:36215 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-27T12:27:27,215 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36215-0x1017c400ae90000, quorum=127.0.0.1:62785, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-27T12:27:27,215 INFO [M:0;1ba5af43fd99:36215 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-27T12:27:27,215 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36215-0x1017c400ae90000, quorum=127.0.0.1:62785, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-27T12:27:27,248 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@70061d7e{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-27T12:27:27,250 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@70195653{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-27T12:27:27,250 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-27T12:27:27,250 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@3ad6a89f{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-27T12:27:27,250 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@15b00a0c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/16bc5e44-ed69-d7f1-cacd-0b5be2f2db9b/hadoop.log.dir/,STOPPED} 2024-11-27T12:27:27,252 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-27T12:27:27,253 WARN [BP-588666114-172.17.0.3-1732710236850 heartbeating to localhost/127.0.0.1:45125 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-27T12:27:27,253 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-27T12:27:27,253 WARN [BP-588666114-172.17.0.3-1732710236850 heartbeating to localhost/127.0.0.1:45125 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-588666114-172.17.0.3-1732710236850 (Datanode Uuid 7c1b600a-6ba2-44c0-98d8-e28cb85bd5bf) service to localhost/127.0.0.1:45125 2024-11-27T12:27:27,255 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/16bc5e44-ed69-d7f1-cacd-0b5be2f2db9b/cluster_2674ffaa-0f8a-9abb-ffed-bcc8bba3b369/data/data1/current/BP-588666114-172.17.0.3-1732710236850 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-27T12:27:27,255 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/16bc5e44-ed69-d7f1-cacd-0b5be2f2db9b/cluster_2674ffaa-0f8a-9abb-ffed-bcc8bba3b369/data/data2/current/BP-588666114-172.17.0.3-1732710236850 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-27T12:27:27,255 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-27T12:27:27,263 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@3f7463d{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-27T12:27:27,264 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@21e93b68{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-27T12:27:27,264 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-27T12:27:27,264 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@2e580bc0{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-27T12:27:27,264 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@1efa708{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/16bc5e44-ed69-d7f1-cacd-0b5be2f2db9b/hadoop.log.dir/,STOPPED} 2024-11-27T12:27:27,282 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-27T12:27:27,397 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down